VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 51271

Last change on this file since 51271 was 51244, checked in by vboxsync, 10 years ago

VMM/HMVMXR0: Relax EFER consistency/corruption checks for the fixes done in r93674.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 490.3 KB
Line 
1/* $Id: HMVMXR0.cpp 51244 2014-05-13 14:19:15Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25#include <iprt/string.h>
26
27#include "HMInternal.h"
28#include <VBox/vmm/vm.h>
29#include "HMVMXR0.h"
30#include <VBox/vmm/pdmapi.h>
31#include <VBox/vmm/dbgf.h>
32#include <VBox/vmm/iem.h>
33#include <VBox/vmm/iom.h>
34#include <VBox/vmm/selm.h>
35#include <VBox/vmm/tm.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_SAVE_FULL_GUEST_STATE
41# define HMVMX_SYNC_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44# define HMVMX_ALWAYS_TRAP_PF
45# define HMVMX_ALWAYS_SWAP_FPU_STATE
46# define HMVMX_ALWAYS_FLUSH_TLB
47# define HMVMX_ALWAYS_SWAP_EFER
48#endif
49
50
51/*******************************************************************************
52* Defined Constants And Macros *
53*******************************************************************************/
54#if defined(RT_ARCH_AMD64)
55# define HMVMX_IS_64BIT_HOST_MODE() (true)
56typedef RTHCUINTREG HMVMXHCUINTREG;
57#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58extern "C" uint32_t g_fVMXIs64bitHost;
59# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
60typedef uint64_t HMVMXHCUINTREG;
61#else
62# define HMVMX_IS_64BIT_HOST_MODE() (false)
63typedef RTHCUINTREG HMVMXHCUINTREG;
64#endif
65
66/** Use the function table. */
67#define HMVMX_USE_FUNCTION_TABLE
68
69/** Determine which tagged-TLB flush handler to use. */
70#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
71#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
72#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
73#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
74
75/** @name Updated-guest-state flags.
76 * @{ */
77#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
78#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
79#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
80#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
81#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
82#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
83#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
84#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
85#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
86#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
87#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
88#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
89#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
90#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
91#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
92#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
93#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
94#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
95#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
96#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
97 | HMVMX_UPDATED_GUEST_RSP \
98 | HMVMX_UPDATED_GUEST_RFLAGS \
99 | HMVMX_UPDATED_GUEST_CR0 \
100 | HMVMX_UPDATED_GUEST_CR3 \
101 | HMVMX_UPDATED_GUEST_CR4 \
102 | HMVMX_UPDATED_GUEST_GDTR \
103 | HMVMX_UPDATED_GUEST_IDTR \
104 | HMVMX_UPDATED_GUEST_LDTR \
105 | HMVMX_UPDATED_GUEST_TR \
106 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
107 | HMVMX_UPDATED_GUEST_DEBUG \
108 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
113 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
114 | HMVMX_UPDATED_GUEST_APIC_STATE)
115/** @} */
116
117/** @name
118 * Flags to skip redundant reads of some common VMCS fields that are not part of
119 * the guest-CPU state but are in the transient structure.
120 */
121#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
122#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
128/** @} */
129
130/** @name
131 * States of the VMCS.
132 *
133 * This does not reflect all possible VMCS states but currently only those
134 * needed for maintaining the VMCS consistently even when thread-context hooks
135 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
136 */
137#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
138#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
139#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
140/** @} */
141
142/**
143 * Exception bitmap mask for real-mode guests (real-on-v86).
144 *
145 * We need to intercept all exceptions manually (except #PF). #NM is also
146 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
147 * even in real-mode if we have Nested Paging support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*******************************************************************************
198* Structures and Typedefs *
199*******************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG uEflags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u6Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringPF;
279} VMXTRANSIENT;
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
284AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
285/** Pointer to VMX transient state. */
286typedef VMXTRANSIENT *PVMXTRANSIENT;
287
288
289/**
290 * MSR-bitmap read permissions.
291 */
292typedef enum VMXMSREXITREAD
293{
294 /** Reading this MSR causes a VM-exit. */
295 VMXMSREXIT_INTERCEPT_READ = 0xb,
296 /** Reading this MSR does not cause a VM-exit. */
297 VMXMSREXIT_PASSTHRU_READ
298} VMXMSREXITREAD;
299/** Pointer to MSR-bitmap read permissions. */
300typedef VMXMSREXITREAD* PVMXMSREXITREAD;
301
302/**
303 * MSR-bitmap write permissions.
304 */
305typedef enum VMXMSREXITWRITE
306{
307 /** Writing to this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
309 /** Writing to this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_WRITE
311} VMXMSREXITWRITE;
312/** Pointer to MSR-bitmap write permissions. */
313typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
314
315
316/**
317 * VMX VM-exit handler.
318 *
319 * @returns VBox status code.
320 * @param pVCpu Pointer to the VMCPU.
321 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
322 * out-of-sync. Make sure to update the required
323 * fields before using them.
324 * @param pVmxTransient Pointer to the VMX-transient structure.
325 */
326#ifndef HMVMX_USE_FUNCTION_TABLE
327typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
328#else
329typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330/** Pointer to VM-exit handler. */
331typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
332#endif
333
334
335/*******************************************************************************
336* Internal Functions *
337*******************************************************************************/
338static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
339static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
340static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
341 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
342#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
343static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
344#endif
345#ifndef HMVMX_USE_FUNCTION_TABLE
346DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
347# define HMVMX_EXIT_DECL static int
348#else
349# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
350#endif
351
352/** @name VM-exit handlers.
353 * @{
354 */
355static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
356static FNVMXEXITHANDLER hmR0VmxExitExtInt;
357static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
358static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
359static FNVMXEXITHANDLER hmR0VmxExitSipi;
360static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
361static FNVMXEXITHANDLER hmR0VmxExitSmi;
362static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
363static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
364static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
365static FNVMXEXITHANDLER hmR0VmxExitCpuid;
366static FNVMXEXITHANDLER hmR0VmxExitGetsec;
367static FNVMXEXITHANDLER hmR0VmxExitHlt;
368static FNVMXEXITHANDLER hmR0VmxExitInvd;
369static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
370static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
371static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
372static FNVMXEXITHANDLER hmR0VmxExitRsm;
373static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
374static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
375static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
376static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
377static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
378static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
379static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
380static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
381static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
382static FNVMXEXITHANDLER hmR0VmxExitMwait;
383static FNVMXEXITHANDLER hmR0VmxExitMtf;
384static FNVMXEXITHANDLER hmR0VmxExitMonitor;
385static FNVMXEXITHANDLER hmR0VmxExitPause;
386static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
387static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
388static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
389static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
391static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
392static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
393static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
394static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
395static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
396static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
397static FNVMXEXITHANDLER hmR0VmxExitRdrand;
398static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
399/** @} */
400
401static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
408static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409#endif
410static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
411
412/*******************************************************************************
413* Global Variables *
414*******************************************************************************/
415#ifdef HMVMX_USE_FUNCTION_TABLE
416
417/**
418 * VMX_EXIT dispatch table.
419 */
420static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
421{
422 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
423 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
424 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
425 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
426 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
427 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
428 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
429 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
430 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
431 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
432 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
433 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
434 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
435 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
436 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
437 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
438 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
439 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
440 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
441 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
442 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
443 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
444 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
445 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
446 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
447 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
448 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
449 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
450 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
451 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
452 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
453 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
454 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
455 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
456 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
457 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
458 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
459 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
460 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
461 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
462 /* 40 UNDEFINED */ hmR0VmxExitPause,
463 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
464 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
465 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
466 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
467 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
468 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
469 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
470 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
471 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
472 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
473 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
474 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
475 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
476 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
477 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
478 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
479 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
480 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
481 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
482};
483#endif /* HMVMX_USE_FUNCTION_TABLE */
484
485#ifdef VBOX_STRICT
486static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
487{
488 /* 0 */ "(Not Used)",
489 /* 1 */ "VMCALL executed in VMX root operation.",
490 /* 2 */ "VMCLEAR with invalid physical address.",
491 /* 3 */ "VMCLEAR with VMXON pointer.",
492 /* 4 */ "VMLAUNCH with non-clear VMCS.",
493 /* 5 */ "VMRESUME with non-launched VMCS.",
494 /* 6 */ "VMRESUME after VMXOFF",
495 /* 7 */ "VM entry with invalid control fields.",
496 /* 8 */ "VM entry with invalid host state fields.",
497 /* 9 */ "VMPTRLD with invalid physical address.",
498 /* 10 */ "VMPTRLD with VMXON pointer.",
499 /* 11 */ "VMPTRLD with incorrect revision identifier.",
500 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
501 /* 13 */ "VMWRITE to read-only VMCS component.",
502 /* 14 */ "(Not Used)",
503 /* 15 */ "VMXON executed in VMX root operation.",
504 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
505 /* 17 */ "VM entry with non-launched executing VMCS.",
506 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
507 /* 19 */ "VMCALL with non-clear VMCS.",
508 /* 20 */ "VMCALL with invalid VM-exit control fields.",
509 /* 21 */ "(Not Used)",
510 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
511 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
512 /* 24 */ "VMCALL with invalid SMM-monitor features.",
513 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
514 /* 26 */ "VM entry with events blocked by MOV SS.",
515 /* 27 */ "(Not Used)",
516 /* 28 */ "Invalid operand to INVEPT/INVVPID."
517};
518#endif /* VBOX_STRICT */
519
520
521
522/**
523 * Updates the VM's last error record. If there was a VMX instruction error,
524 * reads the error data from the VMCS and updates VCPU's last error record as
525 * well.
526 *
527 * @param pVM Pointer to the VM.
528 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
529 * VERR_VMX_UNABLE_TO_START_VM or
530 * VERR_VMX_INVALID_VMCS_FIELD).
531 * @param rc The error code.
532 */
533static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
534{
535 AssertPtr(pVM);
536 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
537 || rc == VERR_VMX_UNABLE_TO_START_VM)
538 {
539 AssertPtrReturnVoid(pVCpu);
540 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
541 }
542 pVM->hm.s.lLastError = rc;
543}
544
545
546/**
547 * Reads the VM-entry interruption-information field from the VMCS into the VMX
548 * transient structure.
549 *
550 * @returns VBox status code.
551 * @param pVmxTransient Pointer to the VMX transient structure.
552 *
553 * @remarks No-long-jump zone!!!
554 */
555DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
556{
557 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
558 AssertRCReturn(rc, rc);
559 return VINF_SUCCESS;
560}
561
562
563/**
564 * Reads the VM-entry exception error code field from the VMCS into
565 * the VMX transient structure.
566 *
567 * @returns VBox status code.
568 * @param pVmxTransient Pointer to the VMX transient structure.
569 *
570 * @remarks No-long-jump zone!!!
571 */
572DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
573{
574 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
575 AssertRCReturn(rc, rc);
576 return VINF_SUCCESS;
577}
578
579
580/**
581 * Reads the VM-entry exception error code field from the VMCS into
582 * the VMX transient structure.
583 *
584 * @returns VBox status code.
585 * @param pVmxTransient Pointer to the VMX transient structure.
586 *
587 * @remarks No-long-jump zone!!!
588 */
589DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
590{
591 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
592 AssertRCReturn(rc, rc);
593 return VINF_SUCCESS;
594}
595
596
597/**
598 * Reads the VM-exit interruption-information field from the VMCS into the VMX
599 * transient structure.
600 *
601 * @returns VBox status code.
602 * @param pVmxTransient Pointer to the VMX transient structure.
603 */
604DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
605{
606 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
607 {
608 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
609 AssertRCReturn(rc, rc);
610 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
611 }
612 return VINF_SUCCESS;
613}
614
615
616/**
617 * Reads the VM-exit interruption error code from the VMCS into the VMX
618 * transient structure.
619 *
620 * @returns VBox status code.
621 * @param pVmxTransient Pointer to the VMX transient structure.
622 */
623DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
624{
625 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
626 {
627 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
628 AssertRCReturn(rc, rc);
629 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
630 }
631 return VINF_SUCCESS;
632}
633
634
635/**
636 * Reads the VM-exit instruction length field from the VMCS into the VMX
637 * transient structure.
638 *
639 * @returns VBox status code.
640 * @param pVCpu Pointer to the VMCPU.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit instruction-information field from the VMCS into
657 * the VMX transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the exit qualification from the VMCS into the VMX transient structure.
676 *
677 * @returns VBox status code.
678 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
679 * case).
680 * @param pVmxTransient Pointer to the VMX transient structure.
681 */
682DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
683{
684 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
685 {
686 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
687 AssertRCReturn(rc, rc);
688 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
689 }
690 return VINF_SUCCESS;
691}
692
693
694/**
695 * Reads the IDT-vectoring information field from the VMCS into the VMX
696 * transient structure.
697 *
698 * @returns VBox status code.
699 * @param pVmxTransient Pointer to the VMX transient structure.
700 *
701 * @remarks No-long-jump zone!!!
702 */
703DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
704{
705 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
706 {
707 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
708 AssertRCReturn(rc, rc);
709 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
710 }
711 return VINF_SUCCESS;
712}
713
714
715/**
716 * Reads the IDT-vectoring error code from the VMCS into the VMX
717 * transient structure.
718 *
719 * @returns VBox status code.
720 * @param pVmxTransient Pointer to the VMX transient structure.
721 */
722DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
723{
724 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
725 {
726 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
727 AssertRCReturn(rc, rc);
728 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
729 }
730 return VINF_SUCCESS;
731}
732
733
734/**
735 * Enters VMX root mode operation on the current CPU.
736 *
737 * @returns VBox status code.
738 * @param pVM Pointer to the VM (optional, can be NULL, after
739 * a resume).
740 * @param HCPhysCpuPage Physical address of the VMXON region.
741 * @param pvCpuPage Pointer to the VMXON region.
742 */
743static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
744{
745 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
746 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
747 Assert(pvCpuPage);
748 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
749
750 if (pVM)
751 {
752 /* Write the VMCS revision dword to the VMXON region. */
753 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
754 }
755
756 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
757 RTCCUINTREG uEflags = ASMIntDisableFlags();
758
759 /* Enable the VMX bit in CR4 if necessary. */
760 RTCCUINTREG uCr4 = ASMGetCR4();
761 if (!(uCr4 & X86_CR4_VMXE))
762 ASMSetCR4(uCr4 | X86_CR4_VMXE);
763
764 /* Enter VMX root mode. */
765 int rc = VMXEnable(HCPhysCpuPage);
766 if (RT_FAILURE(rc))
767 ASMSetCR4(uCr4);
768
769 /* Restore interrupts. */
770 ASMSetFlags(uEflags);
771 return rc;
772}
773
774
775/**
776 * Exits VMX root mode operation on the current CPU.
777 *
778 * @returns VBox status code.
779 */
780static int hmR0VmxLeaveRootMode(void)
781{
782 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
783
784 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
785 RTCCUINTREG uEflags = ASMIntDisableFlags();
786
787 /* If we're for some reason not in VMX root mode, then don't leave it. */
788 RTCCUINTREG uHostCR4 = ASMGetCR4();
789
790 int rc;
791 if (uHostCR4 & X86_CR4_VMXE)
792 {
793 /* Exit VMX root mode and clear the VMX bit in CR4. */
794 VMXDisable();
795 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
796 rc = VINF_SUCCESS;
797 }
798 else
799 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
800
801 /* Restore interrupts. */
802 ASMSetFlags(uEflags);
803 return rc;
804}
805
806
807/**
808 * Allocates and maps one physically contiguous page. The allocated page is
809 * zero'd out. (Used by various VT-x structures).
810 *
811 * @returns IPRT status code.
812 * @param pMemObj Pointer to the ring-0 memory object.
813 * @param ppVirt Where to store the virtual address of the
814 * allocation.
815 * @param pPhys Where to store the physical address of the
816 * allocation.
817 */
818DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
819{
820 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
821 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
822 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
823
824 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
825 if (RT_FAILURE(rc))
826 return rc;
827 *ppVirt = RTR0MemObjAddress(*pMemObj);
828 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
829 ASMMemZero32(*ppVirt, PAGE_SIZE);
830 return VINF_SUCCESS;
831}
832
833
834/**
835 * Frees and unmaps an allocated physical page.
836 *
837 * @param pMemObj Pointer to the ring-0 memory object.
838 * @param ppVirt Where to re-initialize the virtual address of
839 * allocation as 0.
840 * @param pHCPhys Where to re-initialize the physical address of the
841 * allocation as 0.
842 */
843DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
844{
845 AssertPtr(pMemObj);
846 AssertPtr(ppVirt);
847 AssertPtr(pHCPhys);
848 if (*pMemObj != NIL_RTR0MEMOBJ)
849 {
850 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
851 AssertRC(rc);
852 *pMemObj = NIL_RTR0MEMOBJ;
853 *ppVirt = 0;
854 *pHCPhys = 0;
855 }
856}
857
858
859/**
860 * Worker function to free VT-x related structures.
861 *
862 * @returns IPRT status code.
863 * @param pVM Pointer to the VM.
864 */
865static void hmR0VmxStructsFree(PVM pVM)
866{
867 for (VMCPUID i = 0; i < pVM->cCpus; i++)
868 {
869 PVMCPU pVCpu = &pVM->aCpus[i];
870 AssertPtr(pVCpu);
871
872 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
873 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
874
875 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
876 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
877
878 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
879 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
880 }
881
882 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
883#ifdef VBOX_WITH_CRASHDUMP_MAGIC
884 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
885#endif
886}
887
888
889/**
890 * Worker function to allocate VT-x related VM structures.
891 *
892 * @returns IPRT status code.
893 * @param pVM Pointer to the VM.
894 */
895static int hmR0VmxStructsAlloc(PVM pVM)
896{
897 /*
898 * Initialize members up-front so we can cleanup properly on allocation failure.
899 */
900#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
901 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
902 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
903 pVM->hm.s.vmx.HCPhys##a_Name = 0;
904
905#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
906 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
907 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
908 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
909
910#ifdef VBOX_WITH_CRASHDUMP_MAGIC
911 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
912#endif
913 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
914
915 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
916 for (VMCPUID i = 0; i < pVM->cCpus; i++)
917 {
918 PVMCPU pVCpu = &pVM->aCpus[i];
919 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
920 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
921 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
922 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
923 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
924 }
925#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
926#undef VMXLOCAL_INIT_VM_MEMOBJ
927
928 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
929 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
930 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
931 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
932
933 /*
934 * Allocate all the VT-x structures.
935 */
936 int rc = VINF_SUCCESS;
937#ifdef VBOX_WITH_CRASHDUMP_MAGIC
938 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
939 if (RT_FAILURE(rc))
940 goto cleanup;
941 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
942 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
943#endif
944
945 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
946 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
947 {
948 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
949 &pVM->hm.s.vmx.HCPhysApicAccess);
950 if (RT_FAILURE(rc))
951 goto cleanup;
952 }
953
954 /*
955 * Initialize per-VCPU VT-x structures.
956 */
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 AssertPtr(pVCpu);
961
962 /* Allocate the VM control structure (VMCS). */
963 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
964 if (RT_FAILURE(rc))
965 goto cleanup;
966
967 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
968 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
969 {
970 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
971 &pVCpu->hm.s.vmx.HCPhysVirtApic);
972 if (RT_FAILURE(rc))
973 goto cleanup;
974 }
975
976 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
977 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
978 {
979 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
980 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
981 if (RT_FAILURE(rc))
982 goto cleanup;
983 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
984 }
985
986 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
987 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
988 if (RT_FAILURE(rc))
989 goto cleanup;
990
991 /* Allocate the VM-exit MSR-load page for the host MSRs. */
992 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 }
996
997 return VINF_SUCCESS;
998
999cleanup:
1000 hmR0VmxStructsFree(pVM);
1001 return rc;
1002}
1003
1004
1005/**
1006 * Does global VT-x initialization (called during module initialization).
1007 *
1008 * @returns VBox status code.
1009 */
1010VMMR0DECL(int) VMXR0GlobalInit(void)
1011{
1012#ifdef HMVMX_USE_FUNCTION_TABLE
1013 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1014# ifdef VBOX_STRICT
1015 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1016 Assert(g_apfnVMExitHandlers[i]);
1017# endif
1018#endif
1019 return VINF_SUCCESS;
1020}
1021
1022
1023/**
1024 * Does global VT-x termination (called during module termination).
1025 */
1026VMMR0DECL(void) VMXR0GlobalTerm()
1027{
1028 /* Nothing to do currently. */
1029}
1030
1031
1032/**
1033 * Sets up and activates VT-x on the current CPU.
1034 *
1035 * @returns VBox status code.
1036 * @param pCpu Pointer to the global CPU info struct.
1037 * @param pVM Pointer to the VM (can be NULL after a host resume
1038 * operation).
1039 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1040 * fEnabledByHost is true).
1041 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1042 * @a fEnabledByHost is true).
1043 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1044 * enable VT-x on the host.
1045 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1046 */
1047VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1048 void *pvMsrs)
1049{
1050 Assert(pCpu);
1051 Assert(pvMsrs);
1052 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1053
1054 /* Enable VT-x if it's not already enabled by the host. */
1055 if (!fEnabledByHost)
1056 {
1057 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1058 if (RT_FAILURE(rc))
1059 return rc;
1060 }
1061
1062 /*
1063 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1064 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1065 */
1066 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1067 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1068 {
1069 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1070 pCpu->fFlushAsidBeforeUse = false;
1071 }
1072 else
1073 pCpu->fFlushAsidBeforeUse = true;
1074
1075 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1076 ++pCpu->cTlbFlushes;
1077
1078 return VINF_SUCCESS;
1079}
1080
1081
1082/**
1083 * Deactivates VT-x on the current CPU.
1084 *
1085 * @returns VBox status code.
1086 * @param pCpu Pointer to the global CPU info struct.
1087 * @param pvCpuPage Pointer to the VMXON region.
1088 * @param HCPhysCpuPage Physical address of the VMXON region.
1089 *
1090 * @remarks This function should never be called when SUPR0EnableVTx() or
1091 * similar was used to enable VT-x on the host.
1092 */
1093VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1094{
1095 NOREF(pCpu);
1096 NOREF(pvCpuPage);
1097 NOREF(HCPhysCpuPage);
1098
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100 return hmR0VmxLeaveRootMode();
1101}
1102
1103
1104/**
1105 * Sets the permission bits for the specified MSR in the MSR bitmap.
1106 *
1107 * @param pVCpu Pointer to the VMCPU.
1108 * @param uMSR The MSR value.
1109 * @param enmRead Whether reading this MSR causes a VM-exit.
1110 * @param enmWrite Whether writing this MSR causes a VM-exit.
1111 */
1112static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1113{
1114 int32_t iBit;
1115 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1116
1117 /*
1118 * Layout:
1119 * 0x000 - 0x3ff - Low MSR read bits
1120 * 0x400 - 0x7ff - High MSR read bits
1121 * 0x800 - 0xbff - Low MSR write bits
1122 * 0xc00 - 0xfff - High MSR write bits
1123 */
1124 if (uMsr <= 0x00001FFF)
1125 iBit = uMsr;
1126 else if ( uMsr >= 0xC0000000
1127 && uMsr <= 0xC0001FFF)
1128 {
1129 iBit = (uMsr - 0xC0000000);
1130 pbMsrBitmap += 0x400;
1131 }
1132 else
1133 {
1134 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1135 return;
1136 }
1137
1138 Assert(iBit <= 0x1fff);
1139 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1140 ASMBitSet(pbMsrBitmap, iBit);
1141 else
1142 ASMBitClear(pbMsrBitmap, iBit);
1143
1144 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1145 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1146 else
1147 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1148}
1149
1150
1151#ifdef VBOX_STRICT
1152/**
1153 * Gets the permission bits for the specified MSR in the MSR bitmap.
1154 *
1155 * @returns VBox status code.
1156 * @retval VINF_SUCCESS if the specified MSR is found.
1157 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1158 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1159 *
1160 * @param pVCpu Pointer to the VMCPU.
1161 * @param uMsr The MSR.
1162 * @param penmRead Where to store the read permissions.
1163 * @param penmWrite Where to store the write permissions.
1164 */
1165static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1166{
1167 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1168 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1169 int32_t iBit;
1170 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1171
1172 /* See hmR0VmxSetMsrPermission() for the layout. */
1173 if (uMsr <= 0x00001FFF)
1174 iBit = uMsr;
1175 else if ( uMsr >= 0xC0000000
1176 && uMsr <= 0xC0001FFF)
1177 {
1178 iBit = (uMsr - 0xC0000000);
1179 pbMsrBitmap += 0x400;
1180 }
1181 else
1182 {
1183 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1184 return VERR_NOT_SUPPORTED;
1185 }
1186
1187 Assert(iBit <= 0x1fff);
1188 if (ASMBitTest(pbMsrBitmap, iBit))
1189 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1190 else
1191 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1192
1193 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1194 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1195 else
1196 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1197 return VINF_SUCCESS;
1198}
1199#endif /* VBOX_STRICT */
1200
1201
1202/**
1203 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1204 * area.
1205 *
1206 * @returns VBox status code.
1207 * @param pVCpu Pointer to the VMCPU.
1208 * @param cMsrs The number of MSRs.
1209 */
1210DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1211{
1212 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1213 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1214 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1215 {
1216 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1217 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1218 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1219 }
1220
1221 /* Update number of guest MSRs to load/store across the world-switch. */
1222 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1223 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1224
1225 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1226 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1227
1228 /* Update the VCPU's copy of the MSR count. */
1229 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1230
1231 return VINF_SUCCESS;
1232}
1233
1234
1235/**
1236 * Adds a new (or updates the value of an existing) guest/host MSR
1237 * pair to be swapped during the world-switch as part of the
1238 * auto-load/store MSR area in the VMCS.
1239 *
1240 * @returns true if the MSR was added -and- its value was updated, false
1241 * otherwise.
1242 * @param pVCpu Pointer to the VMCPU.
1243 * @param uMsr The MSR.
1244 * @param uGuestMsr Value of the guest MSR.
1245 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1246 * necessary.
1247 */
1248static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1249{
1250 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1251 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1252 uint32_t i;
1253 for (i = 0; i < cMsrs; i++)
1254 {
1255 if (pGuestMsr->u32Msr == uMsr)
1256 break;
1257 pGuestMsr++;
1258 }
1259
1260 bool fAdded = false;
1261 if (i == cMsrs)
1262 {
1263 ++cMsrs;
1264 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1265 AssertRC(rc);
1266
1267 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1268 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1269 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1270
1271 fAdded = true;
1272 }
1273
1274 /* Update the MSR values in the auto-load/store MSR area. */
1275 pGuestMsr->u32Msr = uMsr;
1276 pGuestMsr->u64Value = uGuestMsrValue;
1277
1278 /* Create/update the MSR slot in the host MSR area. */
1279 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1280 pHostMsr += i;
1281 pHostMsr->u32Msr = uMsr;
1282
1283 /*
1284 * Update the host MSR only when requested by the caller AND when we're
1285 * adding it to the auto-load/store area. Otherwise, it would have been
1286 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1287 */
1288 bool fUpdatedMsrValue = false;
1289 if ( fAdded
1290 && fUpdateHostMsr)
1291 {
1292 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1293 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1294 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1295 fUpdatedMsrValue = true;
1296 }
1297
1298 return fUpdatedMsrValue;
1299}
1300
1301
1302/**
1303 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1304 * auto-load/store MSR area in the VMCS.
1305 *
1306 * @returns VBox status code.
1307 * @param pVCpu Pointer to the VMCPU.
1308 * @param uMsr The MSR.
1309 */
1310static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1311{
1312 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1313 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1314 for (uint32_t i = 0; i < cMsrs; i++)
1315 {
1316 /* Find the MSR. */
1317 if (pGuestMsr->u32Msr == uMsr)
1318 {
1319 /* If it's the last MSR, simply reduce the count. */
1320 if (i == cMsrs - 1)
1321 {
1322 --cMsrs;
1323 break;
1324 }
1325
1326 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1327 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1328 pLastGuestMsr += cMsrs - 1;
1329 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1330 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1331
1332 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1333 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1334 pLastHostMsr += cMsrs - 1;
1335 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1336 pHostMsr->u64Value = pLastHostMsr->u64Value;
1337 --cMsrs;
1338 break;
1339 }
1340 pGuestMsr++;
1341 }
1342
1343 /* Update the VMCS if the count changed (meaning the MSR was found). */
1344 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1345 {
1346 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1347 AssertRCReturn(rc, rc);
1348
1349 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1350 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1351 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1352
1353 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1354 return VINF_SUCCESS;
1355 }
1356
1357 return VERR_NOT_FOUND;
1358}
1359
1360
1361/**
1362 * Checks if the specified guest MSR is part of the auto-load/store area in
1363 * the VMCS.
1364 *
1365 * @returns true if found, false otherwise.
1366 * @param pVCpu Pointer to the VMCPU.
1367 * @param uMsr The MSR to find.
1368 */
1369static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1370{
1371 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1372 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1373
1374 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1375 {
1376 if (pGuestMsr->u32Msr == uMsr)
1377 return true;
1378 }
1379 return false;
1380}
1381
1382
1383/**
1384 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1385 *
1386 * @param pVCpu Pointer to the VMCPU.
1387 *
1388 * @remarks No-long-jump zone!!!
1389 */
1390static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1391{
1392 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1393 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1394 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1395 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1396
1397 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1398 {
1399 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1400
1401 /*
1402 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1403 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1404 */
1405 if (pHostMsr->u32Msr == MSR_K6_EFER)
1406 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1407 else
1408 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1409 }
1410
1411 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1412}
1413
1414
1415#if HC_ARCH_BITS == 64
1416/**
1417 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1418 * perform lazy restoration of the host MSRs while leaving VT-x.
1419 *
1420 * @param pVCpu Pointer to the VMCPU.
1421 *
1422 * @remarks No-long-jump zone!!!
1423 */
1424static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1425{
1426 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1427
1428 /*
1429 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1430 */
1431 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1432 {
1433 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1434 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1435 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1436 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1437 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1438 }
1439}
1440
1441
1442/**
1443 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1444 * lazily while leaving VT-x.
1445 *
1446 * @returns true if it does, false otherwise.
1447 * @param pVCpu Pointer to the VMCPU.
1448 * @param uMsr The MSR to check.
1449 */
1450static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1451{
1452 NOREF(pVCpu);
1453 switch (uMsr)
1454 {
1455 case MSR_K8_LSTAR:
1456 case MSR_K6_STAR:
1457 case MSR_K8_SF_MASK:
1458 case MSR_K8_KERNEL_GS_BASE:
1459 return true;
1460 }
1461 return false;
1462}
1463
1464
1465/**
1466 * Saves a set of guests MSRs back into the guest-CPU context.
1467 *
1468 * @param pVCpu Pointer to the VMCPU.
1469 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1470 * out-of-sync. Make sure to update the required fields
1471 * before using them.
1472 *
1473 * @remarks No-long-jump zone!!!
1474 */
1475static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1476{
1477 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1478 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1479
1480 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1481 {
1482 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1483 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1484 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1485 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1486 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1487 }
1488}
1489
1490
1491/**
1492 * Loads a set of guests MSRs to allow read/passthru to the guest.
1493 *
1494 * The name of this function is slightly confusing. This function does NOT
1495 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1496 * common prefix for functions dealing with "lazy restoration" of the shared
1497 * MSRs.
1498 *
1499 * @param pVCpu Pointer to the VMCPU.
1500 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1501 * out-of-sync. Make sure to update the required fields
1502 * before using them.
1503 *
1504 * @remarks No-long-jump zone!!!
1505 */
1506static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1507{
1508 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1509 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1510
1511 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1512 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1513 {
1514#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1515 do { \
1516 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1517 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1518 else \
1519 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1520 } while (0)
1521
1522 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1523 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1524 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1525 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1526#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1527 }
1528 else
1529 {
1530 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1531 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1532 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1533 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1534 }
1535 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1536}
1537
1538
1539/**
1540 * Performs lazy restoration of the set of host MSRs if they were previously
1541 * loaded with guest MSR values.
1542 *
1543 * @param pVCpu Pointer to the VMCPU.
1544 *
1545 * @remarks No-long-jump zone!!!
1546 * @remarks The guest MSRs should have been saved back into the guest-CPU
1547 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1548 */
1549static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1550{
1551 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1552 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1553
1554 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1555 {
1556 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1557 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1558 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1559 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1560 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1561 }
1562 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1563}
1564#endif /* HC_ARCH_BITS == 64 */
1565
1566
1567/**
1568 * Verifies that our cached values of the VMCS controls are all
1569 * consistent with what's actually present in the VMCS.
1570 *
1571 * @returns VBox status code.
1572 * @param pVCpu Pointer to the VMCPU.
1573 */
1574static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1575{
1576 uint32_t u32Val;
1577 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1578 AssertRCReturn(rc, rc);
1579 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1580 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1581
1582 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1583 AssertRCReturn(rc, rc);
1584 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1585 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1586
1587 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1588 AssertRCReturn(rc, rc);
1589 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1590 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1591
1592 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1593 AssertRCReturn(rc, rc);
1594 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1595 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1596
1597 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1598 AssertRCReturn(rc, rc);
1599 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1600 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1601
1602 return VINF_SUCCESS;
1603}
1604
1605
1606#ifdef VBOX_STRICT
1607/**
1608 * Verifies that our cached host EFER value has not changed
1609 * since we cached it.
1610 *
1611 * @param pVCpu Pointer to the VMCPU.
1612 */
1613static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1614{
1615 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1616
1617 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1618 {
1619 uint64_t u64Val;
1620 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1621 AssertRC(rc);
1622
1623 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1624 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1625 }
1626}
1627
1628
1629/**
1630 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1631 * VMCS are correct.
1632 *
1633 * @param pVCpu Pointer to the VMCPU.
1634 */
1635static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1636{
1637 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1638
1639 /* Verify MSR counts in the VMCS are what we think it should be. */
1640 uint32_t cMsrs;
1641 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1642 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1643
1644 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1645 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1646
1647 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1648 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1649
1650 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1651 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1652 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1653 {
1654 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1655 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1656 pGuestMsr->u32Msr, cMsrs));
1657
1658 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1659 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1660 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1661
1662 /* Verify that the permissions are as expected in the MSR bitmap. */
1663 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1664 {
1665 VMXMSREXITREAD enmRead;
1666 VMXMSREXITWRITE enmWrite;
1667 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1668 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1669 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1670 {
1671 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1672 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1673 }
1674 else
1675 {
1676 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1677 pGuestMsr->u32Msr, cMsrs));
1678 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1679 pGuestMsr->u32Msr, cMsrs));
1680 }
1681 }
1682 }
1683}
1684#endif /* VBOX_STRICT */
1685
1686
1687/**
1688 * Flushes the TLB using EPT.
1689 *
1690 * @returns VBox status code.
1691 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1692 * enmFlush).
1693 * @param enmFlush Type of flush.
1694 *
1695 * @remarks Caller is responsible for making sure this function is called only
1696 * when NestedPaging is supported and providing @a enmFlush that is
1697 * supported by the CPU.
1698 * @remarks Can be called with interrupts disabled.
1699 */
1700static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1701{
1702 uint64_t au64Descriptor[2];
1703 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1704 au64Descriptor[0] = 0;
1705 else
1706 {
1707 Assert(pVCpu);
1708 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1709 }
1710 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1711
1712 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1713 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1714 rc));
1715 if ( RT_SUCCESS(rc)
1716 && pVCpu)
1717 {
1718 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1719 }
1720}
1721
1722
1723/**
1724 * Flushes the TLB using VPID.
1725 *
1726 * @returns VBox status code.
1727 * @param pVM Pointer to the VM.
1728 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1729 * enmFlush).
1730 * @param enmFlush Type of flush.
1731 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1732 * on @a enmFlush).
1733 *
1734 * @remarks Can be called with interrupts disabled.
1735 */
1736static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1737{
1738 NOREF(pVM);
1739 AssertPtr(pVM);
1740 Assert(pVM->hm.s.vmx.fVpid);
1741
1742 uint64_t au64Descriptor[2];
1743 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1744 {
1745 au64Descriptor[0] = 0;
1746 au64Descriptor[1] = 0;
1747 }
1748 else
1749 {
1750 AssertPtr(pVCpu);
1751 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1752 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1753 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1754 au64Descriptor[1] = GCPtr;
1755 }
1756
1757 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1758 AssertMsg(rc == VINF_SUCCESS,
1759 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1760 if ( RT_SUCCESS(rc)
1761 && pVCpu)
1762 {
1763 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1764 }
1765}
1766
1767
1768/**
1769 * Invalidates a guest page by guest virtual address. Only relevant for
1770 * EPT/VPID, otherwise there is nothing really to invalidate.
1771 *
1772 * @returns VBox status code.
1773 * @param pVM Pointer to the VM.
1774 * @param pVCpu Pointer to the VMCPU.
1775 * @param GCVirt Guest virtual address of the page to invalidate.
1776 */
1777VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1778{
1779 AssertPtr(pVM);
1780 AssertPtr(pVCpu);
1781 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1782
1783 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1784 if (!fFlushPending)
1785 {
1786 /*
1787 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1788 * See @bugref{6043} and @bugref{6177}.
1789 *
1790 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1791 * function maybe called in a loop with individual addresses.
1792 */
1793 if (pVM->hm.s.vmx.fVpid)
1794 {
1795 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1796 {
1797 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1798 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1799 }
1800 else
1801 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1802 }
1803 else if (pVM->hm.s.fNestedPaging)
1804 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1805 }
1806
1807 return VINF_SUCCESS;
1808}
1809
1810
1811/**
1812 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1813 * otherwise there is nothing really to invalidate.
1814 *
1815 * @returns VBox status code.
1816 * @param pVM Pointer to the VM.
1817 * @param pVCpu Pointer to the VMCPU.
1818 * @param GCPhys Guest physical address of the page to invalidate.
1819 */
1820VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1821{
1822 NOREF(pVM); NOREF(GCPhys);
1823 LogFlowFunc(("%RGp\n", GCPhys));
1824
1825 /*
1826 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1827 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1828 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1829 */
1830 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1831 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1832 return VINF_SUCCESS;
1833}
1834
1835
1836/**
1837 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1838 * case where neither EPT nor VPID is supported by the CPU.
1839 *
1840 * @param pVM Pointer to the VM.
1841 * @param pVCpu Pointer to the VMCPU.
1842 * @param pCpu Pointer to the global HM struct.
1843 *
1844 * @remarks Called with interrupts disabled.
1845 */
1846static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1847{
1848 AssertPtr(pVCpu);
1849 AssertPtr(pCpu);
1850 NOREF(pVM);
1851
1852 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1853
1854 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1855#if 0
1856 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1857 pVCpu->hm.s.TlbShootdown.cPages = 0;
1858#endif
1859
1860 Assert(pCpu->idCpu != NIL_RTCPUID);
1861 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1862 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1863 pVCpu->hm.s.fForceTLBFlush = false;
1864 return;
1865}
1866
1867
1868/**
1869 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1870 *
1871 * @param pVM Pointer to the VM.
1872 * @param pVCpu Pointer to the VMCPU.
1873 * @param pCpu Pointer to the global HM CPU struct.
1874 * @remarks All references to "ASID" in this function pertains to "VPID" in
1875 * Intel's nomenclature. The reason is, to avoid confusion in compare
1876 * statements since the host-CPU copies are named "ASID".
1877 *
1878 * @remarks Called with interrupts disabled.
1879 */
1880static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1881{
1882#ifdef VBOX_WITH_STATISTICS
1883 bool fTlbFlushed = false;
1884# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1885# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1886 if (!fTlbFlushed) \
1887 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1888 } while (0)
1889#else
1890# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1891# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1892#endif
1893
1894 AssertPtr(pVM);
1895 AssertPtr(pCpu);
1896 AssertPtr(pVCpu);
1897 Assert(pCpu->idCpu != NIL_RTCPUID);
1898
1899 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1900 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1901 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1902
1903 /*
1904 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1905 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1906 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1907 */
1908 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1909 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1910 {
1911 ++pCpu->uCurrentAsid;
1912 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1913 {
1914 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1915 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1916 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1917 }
1918
1919 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1920 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1921 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1922
1923 /*
1924 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1925 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1926 */
1927 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1928 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1929 HMVMX_SET_TAGGED_TLB_FLUSHED();
1930 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1931 }
1932
1933 /* Check for explicit TLB shootdowns. */
1934 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1935 {
1936 /*
1937 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1938 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1939 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1940 * but not guest-physical mappings.
1941 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1942 */
1943 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1944 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1945 HMVMX_SET_TAGGED_TLB_FLUSHED();
1946 }
1947
1948 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1949 * where it is commented out. Support individual entry flushing
1950 * someday. */
1951#if 0
1952 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1953 {
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1955
1956 /*
1957 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1958 * as supported by the CPU.
1959 */
1960 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1961 {
1962 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1963 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1964 }
1965 else
1966 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1967
1968 HMVMX_SET_TAGGED_TLB_FLUSHED();
1969 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1970 pVCpu->hm.s.TlbShootdown.cPages = 0;
1971 }
1972#endif
1973
1974 pVCpu->hm.s.fForceTLBFlush = false;
1975
1976 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1977
1978 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1979 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1980 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1981 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1982 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1983 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1984 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1985 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1986 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1987
1988 /* Update VMCS with the VPID. */
1989 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1990 AssertRC(rc);
1991
1992#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1993}
1994
1995
1996/**
1997 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1998 *
1999 * @returns VBox status code.
2000 * @param pVM Pointer to the VM.
2001 * @param pVCpu Pointer to the VMCPU.
2002 * @param pCpu Pointer to the global HM CPU struct.
2003 *
2004 * @remarks Called with interrupts disabled.
2005 */
2006static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2007{
2008 AssertPtr(pVM);
2009 AssertPtr(pVCpu);
2010 AssertPtr(pCpu);
2011 Assert(pCpu->idCpu != NIL_RTCPUID);
2012 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2013 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2014
2015 /*
2016 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2017 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2018 */
2019 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2020 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2021 {
2022 pVCpu->hm.s.fForceTLBFlush = true;
2023 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2024 }
2025
2026 /* Check for explicit TLB shootdown flushes. */
2027 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2028 {
2029 pVCpu->hm.s.fForceTLBFlush = true;
2030 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2031 }
2032
2033 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2034 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2035
2036 if (pVCpu->hm.s.fForceTLBFlush)
2037 {
2038 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2039 pVCpu->hm.s.fForceTLBFlush = false;
2040 }
2041 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2042 * where it is commented out. Support individual entry flushing
2043 * someday. */
2044#if 0
2045 else
2046 {
2047 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2048 {
2049 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2050 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2051 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2052 }
2053 else
2054 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2055
2056 pVCpu->hm.s.TlbShootdown.cPages = 0;
2057 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2058 }
2059#endif
2060}
2061
2062
2063/**
2064 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2065 *
2066 * @returns VBox status code.
2067 * @param pVM Pointer to the VM.
2068 * @param pVCpu Pointer to the VMCPU.
2069 * @param pCpu Pointer to the global HM CPU struct.
2070 *
2071 * @remarks Called with interrupts disabled.
2072 */
2073static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2074{
2075 AssertPtr(pVM);
2076 AssertPtr(pVCpu);
2077 AssertPtr(pCpu);
2078 Assert(pCpu->idCpu != NIL_RTCPUID);
2079 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2080 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2081
2082 /*
2083 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2084 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2085 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2086 */
2087 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2088 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2089 {
2090 pVCpu->hm.s.fForceTLBFlush = true;
2091 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2092 }
2093
2094 /* Check for explicit TLB shootdown flushes. */
2095 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2096 {
2097 /*
2098 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2099 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2100 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2101 */
2102 pVCpu->hm.s.fForceTLBFlush = true;
2103 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2104 }
2105
2106 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2107 if (pVCpu->hm.s.fForceTLBFlush)
2108 {
2109 ++pCpu->uCurrentAsid;
2110 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2111 {
2112 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2113 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2114 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2115 }
2116
2117 pVCpu->hm.s.fForceTLBFlush = false;
2118 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2119 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2120 if (pCpu->fFlushAsidBeforeUse)
2121 {
2122 if (pVM->hm.s.vmx.enmFlushVpid == VMX_FLUSH_VPID_SINGLE_CONTEXT)
2123 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2124 else if (pVM->hm.s.vmx.enmFlushVpid == VMX_FLUSH_VPID_ALL_CONTEXTS)
2125 {
2126 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 /* GCPtr */);
2127 pCpu->fFlushAsidBeforeUse = false;
2128 }
2129 else
2130 {
2131 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2132 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2133 }
2134 }
2135 }
2136 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2137 * where it is commented out. Support individual entry flushing
2138 * someday. */
2139#if 0
2140 else
2141 {
2142 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2143 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2144 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2145 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2146
2147 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2148 {
2149 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2150 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2151 {
2152 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2153 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2154 }
2155 else
2156 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2157
2158 pVCpu->hm.s.TlbShootdown.cPages = 0;
2159 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2160 }
2161 else
2162 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2163 }
2164#endif
2165
2166 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2167 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2168 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2169 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2170 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2171 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2172 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2173
2174 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2175 AssertRC(rc);
2176}
2177
2178
2179/**
2180 * Flushes the guest TLB entry based on CPU capabilities.
2181 *
2182 * @param pVCpu Pointer to the VMCPU.
2183 * @param pCpu Pointer to the global HM CPU struct.
2184 */
2185DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2186{
2187#ifdef HMVMX_ALWAYS_FLUSH_TLB
2188 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2189#endif
2190 PVM pVM = pVCpu->CTX_SUFF(pVM);
2191 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2192 {
2193 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2194 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2195 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2196 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2197 default:
2198 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2199 break;
2200 }
2201
2202 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2203 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2204
2205 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2206}
2207
2208
2209/**
2210 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2211 * TLB entries from the host TLB before VM-entry.
2212 *
2213 * @returns VBox status code.
2214 * @param pVM Pointer to the VM.
2215 */
2216static int hmR0VmxSetupTaggedTlb(PVM pVM)
2217{
2218 /*
2219 * Determine optimal flush type for Nested Paging.
2220 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2221 * guest execution (see hmR3InitFinalizeR0()).
2222 */
2223 if (pVM->hm.s.fNestedPaging)
2224 {
2225 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2226 {
2227 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2228 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2229 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2230 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2231 else
2232 {
2233 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2234 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237
2238 /* Make sure the write-back cacheable memory type for EPT is supported. */
2239 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2240 {
2241 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2242 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246 else
2247 {
2248 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2249 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2250 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2251 }
2252 }
2253
2254 /*
2255 * Determine optimal flush type for VPID.
2256 */
2257 if (pVM->hm.s.vmx.fVpid)
2258 {
2259 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2260 {
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2262 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2263 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2264 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2265 else
2266 {
2267 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2268 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2269 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2270 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2271 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2272 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2273 pVM->hm.s.vmx.fVpid = false;
2274 }
2275 }
2276 else
2277 {
2278 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2279 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2280 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2281 pVM->hm.s.vmx.fVpid = false;
2282 }
2283 }
2284
2285 /*
2286 * Setup the handler for flushing tagged-TLBs.
2287 */
2288 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2289 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2290 else if (pVM->hm.s.fNestedPaging)
2291 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2292 else if (pVM->hm.s.vmx.fVpid)
2293 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2294 else
2295 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2296 return VINF_SUCCESS;
2297}
2298
2299
2300/**
2301 * Sets up pin-based VM-execution controls in the VMCS.
2302 *
2303 * @returns VBox status code.
2304 * @param pVM Pointer to the VM.
2305 * @param pVCpu Pointer to the VMCPU.
2306 */
2307static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2308{
2309 AssertPtr(pVM);
2310 AssertPtr(pVCpu);
2311
2312 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2313 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2314
2315 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2316 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2317 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2318
2319 /* Enable the VMX preemption timer. */
2320 if (pVM->hm.s.vmx.fUsePreemptTimer)
2321 {
2322 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2323 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2324 }
2325
2326 if ((val & zap) != val)
2327 {
2328 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2329 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2330 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2331 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2332 }
2333
2334 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2335 AssertRCReturn(rc, rc);
2336
2337 /* Update VCPU with the currently set pin-based VM-execution controls. */
2338 pVCpu->hm.s.vmx.u32PinCtls = val;
2339 return rc;
2340}
2341
2342
2343/**
2344 * Sets up processor-based VM-execution controls in the VMCS.
2345 *
2346 * @returns VBox status code.
2347 * @param pVM Pointer to the VM.
2348 * @param pVMCPU Pointer to the VMCPU.
2349 */
2350static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2351{
2352 AssertPtr(pVM);
2353 AssertPtr(pVCpu);
2354
2355 int rc = VERR_INTERNAL_ERROR_5;
2356 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2357 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2358
2359 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2360 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2361 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2362 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2363 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2364 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2365 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2366
2367 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2368 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2369 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2370 {
2371 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2372 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2373 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2374 }
2375
2376 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2377 if (!pVM->hm.s.fNestedPaging)
2378 {
2379 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2380 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2381 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2382 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2383 }
2384
2385 /* Use TPR shadowing if supported by the CPU. */
2386 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2387 {
2388 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2389 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2390 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2391 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2392 AssertRCReturn(rc, rc);
2393
2394 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2395 /* CR8 writes causes a VM-exit based on TPR threshold. */
2396 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2397 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2398 }
2399 else
2400 {
2401 /*
2402 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2403 * Set this control only for 64-bit guests.
2404 */
2405 if (pVM->hm.s.fAllow64BitGuests)
2406 {
2407 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2408 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2409 }
2410 }
2411
2412 /* Use MSR-bitmaps if supported by the CPU. */
2413 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2414 {
2415 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2416
2417 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2418 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2419 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2420 AssertRCReturn(rc, rc);
2421
2422 /*
2423 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2424 * automatically as dedicated fields in the VMCS.
2425 */
2426 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2427 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2428 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2429 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2430 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2431
2432#if HC_ARCH_BITS == 64
2433 /*
2434 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2435 */
2436 if (pVM->hm.s.fAllow64BitGuests)
2437 {
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 }
2443#endif
2444 }
2445
2446 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2447 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2448 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2449
2450 if ((val & zap) != val)
2451 {
2452 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2453 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2454 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2455 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2456 }
2457
2458 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2459 AssertRCReturn(rc, rc);
2460
2461 /* Update VCPU with the currently set processor-based VM-execution controls. */
2462 pVCpu->hm.s.vmx.u32ProcCtls = val;
2463
2464 /*
2465 * Secondary processor-based VM-execution controls.
2466 */
2467 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2468 {
2469 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2470 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2471
2472 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2473 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2474
2475 if (pVM->hm.s.fNestedPaging)
2476 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2477 else
2478 {
2479 /*
2480 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2481 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2482 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2483 */
2484 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2485 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2486 }
2487
2488 if (pVM->hm.s.vmx.fVpid)
2489 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2490
2491 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2492 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2493
2494 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2495 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2496 * done dynamically. */
2497 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2498 {
2499 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2500 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2501 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2502 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2503 AssertRCReturn(rc, rc);
2504 }
2505
2506 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2507 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2508
2509 if ((val & zap) != val)
2510 {
2511 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2512 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2513 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2514 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2515 }
2516
2517 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2518 AssertRCReturn(rc, rc);
2519
2520 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2521 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2522 }
2523 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2524 {
2525 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2526 "available\n"));
2527 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2528 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2529 }
2530
2531 return VINF_SUCCESS;
2532}
2533
2534
2535/**
2536 * Sets up miscellaneous (everything other than Pin & Processor-based
2537 * VM-execution) control fields in the VMCS.
2538 *
2539 * @returns VBox status code.
2540 * @param pVM Pointer to the VM.
2541 * @param pVCpu Pointer to the VMCPU.
2542 */
2543static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2544{
2545 NOREF(pVM);
2546 AssertPtr(pVM);
2547 AssertPtr(pVCpu);
2548
2549 int rc = VERR_GENERAL_FAILURE;
2550
2551 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2552#if 0
2553 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2554 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2555 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2556
2557 /*
2558 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2559 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2560 * We thus use the exception bitmap to control it rather than use both.
2561 */
2562 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2563 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2564
2565 /** @todo Explore possibility of using IO-bitmaps. */
2566 /* All IO & IOIO instructions cause VM-exits. */
2567 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2568 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2569
2570 /* Initialize the MSR-bitmap area. */
2571 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2572 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2574#endif
2575
2576 /* Setup MSR auto-load/store area. */
2577 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2578 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2579 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2580 AssertRCReturn(rc, rc);
2581 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2582 AssertRCReturn(rc, rc);
2583
2584 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2585 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2586 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2587 AssertRCReturn(rc, rc);
2588
2589 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2590 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2591 AssertRCReturn(rc, rc);
2592
2593 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2594#if 0
2595 /* Setup debug controls */
2596 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2597 AssertRCReturn(rc, rc);
2598 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2599 AssertRCReturn(rc, rc);
2600#endif
2601
2602 return rc;
2603}
2604
2605
2606/**
2607 * Sets up the initial exception bitmap in the VMCS based on static conditions
2608 * (i.e. conditions that cannot ever change after starting the VM).
2609 *
2610 * @returns VBox status code.
2611 * @param pVM Pointer to the VM.
2612 * @param pVCpu Pointer to the VMCPU.
2613 */
2614static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2615{
2616 AssertPtr(pVM);
2617 AssertPtr(pVCpu);
2618
2619 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2620
2621 uint32_t u32XcptBitmap = 0;
2622
2623 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2624 if (!pVM->hm.s.fNestedPaging)
2625 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2626
2627 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2628 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2629 AssertRCReturn(rc, rc);
2630 return rc;
2631}
2632
2633
2634/**
2635 * Sets up the initial guest-state mask. The guest-state mask is consulted
2636 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2637 * for the nested virtualization case (as it would cause a VM-exit).
2638 *
2639 * @param pVCpu Pointer to the VMCPU.
2640 */
2641static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2642{
2643 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2644 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2645 return VINF_SUCCESS;
2646}
2647
2648
2649/**
2650 * Does per-VM VT-x initialization.
2651 *
2652 * @returns VBox status code.
2653 * @param pVM Pointer to the VM.
2654 */
2655VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2656{
2657 LogFlowFunc(("pVM=%p\n", pVM));
2658
2659 int rc = hmR0VmxStructsAlloc(pVM);
2660 if (RT_FAILURE(rc))
2661 {
2662 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2663 return rc;
2664 }
2665
2666 return VINF_SUCCESS;
2667}
2668
2669
2670/**
2671 * Does per-VM VT-x termination.
2672 *
2673 * @returns VBox status code.
2674 * @param pVM Pointer to the VM.
2675 */
2676VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2677{
2678 LogFlowFunc(("pVM=%p\n", pVM));
2679
2680#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2681 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2682 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2683#endif
2684 hmR0VmxStructsFree(pVM);
2685 return VINF_SUCCESS;
2686}
2687
2688
2689/**
2690 * Sets up the VM for execution under VT-x.
2691 * This function is only called once per-VM during initialization.
2692 *
2693 * @returns VBox status code.
2694 * @param pVM Pointer to the VM.
2695 */
2696VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2697{
2698 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2699 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2700
2701 LogFlowFunc(("pVM=%p\n", pVM));
2702
2703 /*
2704 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2705 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2706 */
2707 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2708 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2709 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2710 || !pVM->hm.s.vmx.pRealModeTSS))
2711 {
2712 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2713 return VERR_INTERNAL_ERROR;
2714 }
2715
2716#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2717 /*
2718 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2719 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2720 */
2721 if ( pVM->hm.s.fAllow64BitGuests
2722 && !HMVMX_IS_64BIT_HOST_MODE())
2723 {
2724 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2725 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2726 }
2727#endif
2728
2729 /* Initialize these always, see hmR3InitFinalizeR0().*/
2730 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2731 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2732
2733 /* Setup the tagged-TLB flush handlers. */
2734 int rc = hmR0VmxSetupTaggedTlb(pVM);
2735 if (RT_FAILURE(rc))
2736 {
2737 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2738 return rc;
2739 }
2740
2741 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2742 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2743#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2744 if ( HMVMX_IS_64BIT_HOST_MODE()
2745 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2746 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2747 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2748 {
2749 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2750 }
2751#endif
2752
2753 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2754 {
2755 PVMCPU pVCpu = &pVM->aCpus[i];
2756 AssertPtr(pVCpu);
2757 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2758
2759 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2760 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2761
2762 /* Set revision dword at the beginning of the VMCS structure. */
2763 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2764
2765 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2766 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2767 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2768 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2769
2770 /* Load this VMCS as the current VMCS. */
2771 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2772 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2773 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2774
2775 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2776 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2777 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2778
2779 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2780 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2781 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2782
2783 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2784 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2785 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2786
2787 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2788 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2789 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2790
2791 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2793 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2794
2795#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2796 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2798 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2799#endif
2800
2801 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2802 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2807
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2809 }
2810
2811 return VINF_SUCCESS;
2812}
2813
2814
2815/**
2816 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2817 * the VMCS.
2818 *
2819 * @returns VBox status code.
2820 * @param pVM Pointer to the VM.
2821 * @param pVCpu Pointer to the VMCPU.
2822 */
2823DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2824{
2825 NOREF(pVM); NOREF(pVCpu);
2826
2827 RTCCUINTREG uReg = ASMGetCR0();
2828 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2829 AssertRCReturn(rc, rc);
2830
2831#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2832 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2833 if (HMVMX_IS_64BIT_HOST_MODE())
2834 {
2835 uint64_t uRegCR3 = HMR0Get64bitCR3();
2836 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2837 }
2838 else
2839#endif
2840 {
2841 uReg = ASMGetCR3();
2842 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2843 }
2844 AssertRCReturn(rc, rc);
2845
2846 uReg = ASMGetCR4();
2847 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2848 AssertRCReturn(rc, rc);
2849 return rc;
2850}
2851
2852
2853#if HC_ARCH_BITS == 64
2854/**
2855 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2856 * requirements. See hmR0VmxSaveHostSegmentRegs().
2857 */
2858# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2859 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2860 { \
2861 bool fValidSelector = true; \
2862 if ((selValue) & X86_SEL_LDT) \
2863 { \
2864 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2865 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2866 } \
2867 if (fValidSelector) \
2868 { \
2869 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2870 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2871 } \
2872 (selValue) = 0; \
2873 }
2874#endif
2875
2876
2877/**
2878 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2879 * the host-state area in the VMCS.
2880 *
2881 * @returns VBox status code.
2882 * @param pVM Pointer to the VM.
2883 * @param pVCpu Pointer to the VMCPU.
2884 */
2885DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2886{
2887 NOREF(pVM);
2888 int rc = VERR_INTERNAL_ERROR_5;
2889
2890#if HC_ARCH_BITS == 64
2891 /*
2892 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2893 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2894 */
2895 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2896 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2897#endif
2898
2899 /*
2900 * Host DS, ES, FS and GS segment registers.
2901 */
2902#if HC_ARCH_BITS == 64
2903 RTSEL uSelDS = ASMGetDS();
2904 RTSEL uSelES = ASMGetES();
2905 RTSEL uSelFS = ASMGetFS();
2906 RTSEL uSelGS = ASMGetGS();
2907#else
2908 RTSEL uSelDS = 0;
2909 RTSEL uSelES = 0;
2910 RTSEL uSelFS = 0;
2911 RTSEL uSelGS = 0;
2912#endif
2913
2914 /* Recalculate which host-state bits need to be manually restored. */
2915 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2916
2917 /*
2918 * Host CS and SS segment registers.
2919 */
2920#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2921 RTSEL uSelCS;
2922 RTSEL uSelSS;
2923 if (HMVMX_IS_64BIT_HOST_MODE())
2924 {
2925 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2926 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2927 }
2928 else
2929 {
2930 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2931 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2932 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2933 }
2934#else
2935 RTSEL uSelCS = ASMGetCS();
2936 RTSEL uSelSS = ASMGetSS();
2937#endif
2938
2939 /*
2940 * Host TR segment register.
2941 */
2942 RTSEL uSelTR = ASMGetTR();
2943
2944#if HC_ARCH_BITS == 64
2945 /*
2946 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2947 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2948 */
2949 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2950 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2951 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2952 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2953# undef VMXLOCAL_ADJUST_HOST_SEG
2954#endif
2955
2956 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2957 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2958 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2959 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2960 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2961 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2962 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2963 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2964 Assert(uSelCS);
2965 Assert(uSelTR);
2966
2967 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2968#if 0
2969 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2970 Assert(uSelSS != 0);
2971#endif
2972
2973 /* Write these host selector fields into the host-state area in the VMCS. */
2974 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2975 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2976#if HC_ARCH_BITS == 64
2977 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2978 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2979 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2980 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2981#endif
2982 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2983
2984 /*
2985 * Host GDTR and IDTR.
2986 */
2987 RTGDTR Gdtr;
2988 RT_ZERO(Gdtr);
2989#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2990 if (HMVMX_IS_64BIT_HOST_MODE())
2991 {
2992 X86XDTR64 Gdtr64;
2993 X86XDTR64 Idtr64;
2994 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2995 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2997
2998 Gdtr.cbGdt = Gdtr64.cb;
2999 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3000 }
3001 else
3002#endif
3003 {
3004 RTIDTR Idtr;
3005 ASMGetGDTR(&Gdtr);
3006 ASMGetIDTR(&Idtr);
3007 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3008 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3009
3010#if HC_ARCH_BITS == 64
3011 /*
3012 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3013 * maximum limit (0xffff) on every VM-exit.
3014 */
3015 if (Gdtr.cbGdt != 0xffff)
3016 {
3017 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3018 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3019 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3020 }
3021
3022 /*
3023 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3024 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3025 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3026 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3027 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3028 * hosts where we are pretty sure it won't cause trouble.
3029 */
3030# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3031 if (Idtr.cbIdt < 0x0fff)
3032# else
3033 if (Idtr.cbIdt != 0xffff)
3034# endif
3035 {
3036 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3037 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3038 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3039 }
3040#endif
3041 }
3042
3043 /*
3044 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3045 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3046 */
3047 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3048 {
3049 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3050 return VERR_VMX_INVALID_HOST_STATE;
3051 }
3052
3053 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3054#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3055 if (HMVMX_IS_64BIT_HOST_MODE())
3056 {
3057 /* We need the 64-bit TR base for hybrid darwin. */
3058 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3059 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3060 }
3061 else
3062#endif
3063 {
3064 uintptr_t uTRBase;
3065#if HC_ARCH_BITS == 64
3066 uTRBase = X86DESC64_BASE(pDesc);
3067
3068 /*
3069 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3070 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3071 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3072 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3073 *
3074 * [1] See Intel spec. 3.5 "System Descriptor Types".
3075 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3076 */
3077 Assert(pDesc->System.u4Type == 11);
3078 if ( pDesc->System.u16LimitLow != 0x67
3079 || pDesc->System.u4LimitHigh)
3080 {
3081 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3082 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3083
3084 /* Store the GDTR here as we need it while restoring TR. */
3085 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3086 }
3087#else
3088 uTRBase = X86DESC_BASE(pDesc);
3089#endif
3090 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3091 }
3092 AssertRCReturn(rc, rc);
3093
3094 /*
3095 * Host FS base and GS base.
3096 */
3097#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3098 if (HMVMX_IS_64BIT_HOST_MODE())
3099 {
3100 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3101 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3102 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3103 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3104
3105# if HC_ARCH_BITS == 64
3106 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3107 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3108 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3109 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3110 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3111# endif
3112 }
3113#endif
3114 return rc;
3115}
3116
3117
3118/**
3119 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3120 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3121 * the host after every successful VM-exit.
3122 *
3123 * @returns VBox status code.
3124 * @param pVM Pointer to the VM.
3125 * @param pVCpu Pointer to the VMCPU.
3126 *
3127 * @remarks No-long-jump zone!!!
3128 */
3129DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3130{
3131 NOREF(pVM);
3132
3133 AssertPtr(pVCpu);
3134 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3135
3136 int rc = VINF_SUCCESS;
3137#if HC_ARCH_BITS == 64
3138 if (pVM->hm.s.fAllow64BitGuests)
3139 hmR0VmxLazySaveHostMsrs(pVCpu);
3140#endif
3141
3142 /*
3143 * Host Sysenter MSRs.
3144 */
3145 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3146 AssertRCReturn(rc, rc);
3147#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3148 if (HMVMX_IS_64BIT_HOST_MODE())
3149 {
3150 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3151 AssertRCReturn(rc, rc);
3152 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3153 }
3154 else
3155 {
3156 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3157 AssertRCReturn(rc, rc);
3158 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3159 }
3160#elif HC_ARCH_BITS == 32
3161 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3162 AssertRCReturn(rc, rc);
3163 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3164#else
3165 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3166 AssertRCReturn(rc, rc);
3167 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3168#endif
3169 AssertRCReturn(rc, rc);
3170
3171 /*
3172 * Host EFER MSR.
3173 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3174 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3175 */
3176 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3177 {
3178 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3179 AssertRCReturn(rc, rc);
3180 }
3181
3182 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3183 * hmR0VmxLoadGuestExitCtls() !! */
3184
3185 return rc;
3186}
3187
3188
3189/**
3190 * Figures out if we need to swap the EFER MSR which is
3191 * particularly expensive.
3192 *
3193 * We check all relevant bits. For now, that's everything
3194 * besides LMA/LME, as these two bits are handled by VM-entry,
3195 * see hmR0VmxLoadGuestExitCtls() and
3196 * hmR0VMxLoadGuestEntryCtls().
3197 *
3198 * @returns true if we need to load guest EFER, false otherwise.
3199 * @param pVCpu Pointer to the VMCPU.
3200 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3201 * out-of-sync. Make sure to update the required fields
3202 * before using them.
3203 *
3204 * @remarks Requires EFER, CR4.
3205 * @remarks No-long-jump zone!!!
3206 */
3207static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3208{
3209#ifdef HMVMX_ALWAYS_SWAP_EFER
3210 return true;
3211#endif
3212
3213#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3214 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3215 if (CPUMIsGuestInLongMode(pVCpu))
3216 return false;
3217#endif
3218
3219 PVM pVM = pVCpu->CTX_SUFF(pVM);
3220 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3221 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3222
3223 /*
3224 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3225 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3226 */
3227 if ( CPUMIsGuestInLongMode(pVCpu)
3228 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3229 {
3230 return true;
3231 }
3232
3233 /*
3234 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3235 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3236 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3237 */
3238 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3239 && (pMixedCtx->cr0 & X86_CR0_PG)
3240 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3241 {
3242 /* Assert that host is PAE capable. */
3243 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3244 return true;
3245 }
3246
3247 /** @todo Check the latest Intel spec. for any other bits,
3248 * like SMEP/SMAP? */
3249 return false;
3250}
3251
3252
3253/**
3254 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3255 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3256 * controls".
3257 *
3258 * @returns VBox status code.
3259 * @param pVCpu Pointer to the VMCPU.
3260 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3261 * out-of-sync. Make sure to update the required fields
3262 * before using them.
3263 *
3264 * @remarks Requires EFER.
3265 * @remarks No-long-jump zone!!!
3266 */
3267DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3268{
3269 int rc = VINF_SUCCESS;
3270 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3271 {
3272 PVM pVM = pVCpu->CTX_SUFF(pVM);
3273 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3274 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3275
3276 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3277 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3278
3279 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3280 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3281 {
3282 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3283 Log4(("Load: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3284 }
3285 else
3286 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3287
3288 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3289 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3290 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3291 {
3292 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3293 Log4(("Load: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3294 }
3295
3296 /*
3297 * The following should -not- be set (since we're not in SMM mode):
3298 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3299 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3300 */
3301
3302 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3303 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3304
3305 if ((val & zap) != val)
3306 {
3307 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3308 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3309 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3310 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3311 }
3312
3313 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3314 AssertRCReturn(rc, rc);
3315
3316 /* Update VCPU with the currently set VM-exit controls. */
3317 pVCpu->hm.s.vmx.u32EntryCtls = val;
3318 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3319 }
3320 return rc;
3321}
3322
3323
3324/**
3325 * Sets up the VM-exit controls in the VMCS.
3326 *
3327 * @returns VBox status code.
3328 * @param pVM Pointer to the VM.
3329 * @param pVCpu Pointer to the VMCPU.
3330 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3331 * out-of-sync. Make sure to update the required fields
3332 * before using them.
3333 *
3334 * @remarks Requires EFER.
3335 */
3336DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3337{
3338 NOREF(pMixedCtx);
3339
3340 int rc = VINF_SUCCESS;
3341 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3342 {
3343 PVM pVM = pVCpu->CTX_SUFF(pVM);
3344 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3345 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3346
3347 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3348 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3349
3350 /*
3351 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3352 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3353 */
3354#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3355 if (HMVMX_IS_64BIT_HOST_MODE())
3356 {
3357 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3358 Log4(("Load: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3359 }
3360 else
3361 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3362#else
3363 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3364 {
3365 /* The switcher returns to long mode, EFER is managed by the switcher. */
3366 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3367 Log4(("Load: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3368 }
3369 else
3370 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3371#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3372
3373 /* If the newer VMCS fields for managing EFER exists, use it. */
3374 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3375 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3376 {
3377 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3378 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3379 Log4(("Load: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3380 }
3381
3382 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3383 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3384
3385 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3386 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3387 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3388
3389 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3390 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3391
3392 if ((val & zap) != val)
3393 {
3394 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3395 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3396 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3397 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3398 }
3399
3400 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3401 AssertRCReturn(rc, rc);
3402
3403 /* Update VCPU with the currently set VM-exit controls. */
3404 pVCpu->hm.s.vmx.u32ExitCtls = val;
3405 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3406 }
3407 return rc;
3408}
3409
3410
3411/**
3412 * Loads the guest APIC and related state.
3413 *
3414 * @returns VBox status code.
3415 * @param pVM Pointer to the VM.
3416 * @param pVCpu Pointer to the VMCPU.
3417 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3418 * out-of-sync. Make sure to update the required fields
3419 * before using them.
3420 */
3421DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3422{
3423 NOREF(pMixedCtx);
3424
3425 int rc = VINF_SUCCESS;
3426 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3427 {
3428 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3429 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3430 {
3431 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3432
3433 bool fPendingIntr = false;
3434 uint8_t u8Tpr = 0;
3435 uint8_t u8PendingIntr = 0;
3436 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3437 AssertRCReturn(rc, rc);
3438
3439 /*
3440 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3441 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3442 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3443 * the interrupt when we VM-exit for other reasons.
3444 */
3445 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3446 uint32_t u32TprThreshold = 0;
3447 if (fPendingIntr)
3448 {
3449 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3450 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3451 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3452 if (u8PendingPriority <= u8TprPriority)
3453 u32TprThreshold = u8PendingPriority;
3454 else
3455 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3456 }
3457 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3458
3459 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3460 AssertRCReturn(rc, rc);
3461 }
3462
3463 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3464 }
3465 return rc;
3466}
3467
3468
3469/**
3470 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3471 *
3472 * @returns Guest's interruptibility-state.
3473 * @param pVCpu Pointer to the VMCPU.
3474 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3475 * out-of-sync. Make sure to update the required fields
3476 * before using them.
3477 *
3478 * @remarks No-long-jump zone!!!
3479 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3480 */
3481DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3482{
3483 /*
3484 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3485 * inhibit interrupts or clear any existing interrupt-inhibition.
3486 */
3487 uint32_t uIntrState = 0;
3488 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3489 {
3490 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3491 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3492 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3493 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3494 {
3495 /*
3496 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3497 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3498 */
3499 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3500 }
3501 else if (pMixedCtx->eflags.Bits.u1IF)
3502 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3503 else
3504 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3505 }
3506 return uIntrState;
3507}
3508
3509
3510/**
3511 * Loads the guest's interruptibility-state into the guest-state area in the
3512 * VMCS.
3513 *
3514 * @returns VBox status code.
3515 * @param pVCpu Pointer to the VMCPU.
3516 * @param uIntrState The interruptibility-state to set.
3517 */
3518static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3519{
3520 NOREF(pVCpu);
3521 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3522 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3523 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3524 AssertRCReturn(rc, rc);
3525 return rc;
3526}
3527
3528
3529/**
3530 * Loads the guest's RIP into the guest-state area in the VMCS.
3531 *
3532 * @returns VBox status code.
3533 * @param pVCpu Pointer to the VMCPU.
3534 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3535 * out-of-sync. Make sure to update the required fields
3536 * before using them.
3537 *
3538 * @remarks No-long-jump zone!!!
3539 */
3540static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3541{
3542 int rc = VINF_SUCCESS;
3543 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3544 {
3545 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3546 AssertRCReturn(rc, rc);
3547
3548 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3549 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3550 }
3551 return rc;
3552}
3553
3554
3555/**
3556 * Loads the guest's RSP into the guest-state area in the VMCS.
3557 *
3558 * @returns VBox status code.
3559 * @param pVCpu Pointer to the VMCPU.
3560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3561 * out-of-sync. Make sure to update the required fields
3562 * before using them.
3563 *
3564 * @remarks No-long-jump zone!!!
3565 */
3566static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3567{
3568 int rc = VINF_SUCCESS;
3569 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3570 {
3571 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3572 AssertRCReturn(rc, rc);
3573
3574 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3575 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3576 }
3577 return rc;
3578}
3579
3580
3581/**
3582 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3583 *
3584 * @returns VBox status code.
3585 * @param pVCpu Pointer to the VMCPU.
3586 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3587 * out-of-sync. Make sure to update the required fields
3588 * before using them.
3589 *
3590 * @remarks No-long-jump zone!!!
3591 */
3592static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3593{
3594 int rc = VINF_SUCCESS;
3595 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3596 {
3597 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3598 Let us assert it as such and use 32-bit VMWRITE. */
3599 Assert(!(pMixedCtx->rflags.u64 >> 32));
3600 X86EFLAGS Eflags = pMixedCtx->eflags;
3601 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3602 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3603
3604 /*
3605 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3606 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3607 */
3608 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3609 {
3610 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3611 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3612 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3613 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3614 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3615 }
3616
3617 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3618 AssertRCReturn(rc, rc);
3619
3620 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3621 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3622 }
3623 return rc;
3624}
3625
3626
3627/**
3628 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3629 *
3630 * @returns VBox status code.
3631 * @param pVCpu Pointer to the VMCPU.
3632 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3633 * out-of-sync. Make sure to update the required fields
3634 * before using them.
3635 *
3636 * @remarks No-long-jump zone!!!
3637 */
3638DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3639{
3640 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3641 AssertRCReturn(rc, rc);
3642 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3643 AssertRCReturn(rc, rc);
3644 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3645 AssertRCReturn(rc, rc);
3646 return rc;
3647}
3648
3649
3650/**
3651 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3652 * CR0 is partially shared with the host and we have to consider the FPU bits.
3653 *
3654 * @returns VBox status code.
3655 * @param pVM Pointer to the VM.
3656 * @param pVCpu Pointer to the VMCPU.
3657 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3658 * out-of-sync. Make sure to update the required fields
3659 * before using them.
3660 *
3661 * @remarks No-long-jump zone!!!
3662 */
3663static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3664{
3665 /*
3666 * Guest CR0.
3667 * Guest FPU.
3668 */
3669 int rc = VINF_SUCCESS;
3670 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3671 {
3672 Assert(!(pMixedCtx->cr0 >> 32));
3673 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3674 PVM pVM = pVCpu->CTX_SUFF(pVM);
3675
3676 /* The guest's view (read access) of its CR0 is unblemished. */
3677 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3678 AssertRCReturn(rc, rc);
3679 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3680
3681 /* Setup VT-x's view of the guest CR0. */
3682 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3683 if (pVM->hm.s.fNestedPaging)
3684 {
3685 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3686 {
3687 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3688 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3689 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3690 }
3691 else
3692 {
3693 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3694 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3695 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3696 }
3697
3698 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3699 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3700 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3701
3702 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3703 AssertRCReturn(rc, rc);
3704 }
3705 else
3706 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3707
3708 /*
3709 * Guest FPU bits.
3710 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3711 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3712 */
3713 u32GuestCR0 |= X86_CR0_NE;
3714 bool fInterceptNM = false;
3715 if (CPUMIsGuestFPUStateActive(pVCpu))
3716 {
3717 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3718 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3719 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3720 }
3721 else
3722 {
3723 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3724 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3725 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3726 }
3727
3728 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3729 bool fInterceptMF = false;
3730 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3731 fInterceptMF = true;
3732
3733 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3734 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3735 {
3736 Assert(PDMVmmDevHeapIsEnabled(pVM));
3737 Assert(pVM->hm.s.vmx.pRealModeTSS);
3738 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3739 fInterceptNM = true;
3740 fInterceptMF = true;
3741 }
3742 else
3743 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3744
3745 if (fInterceptNM)
3746 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3747 else
3748 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3749
3750 if (fInterceptMF)
3751 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3752 else
3753 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3754
3755 /* Additional intercepts for debugging, define these yourself explicitly. */
3756#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3757 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3758 | RT_BIT(X86_XCPT_BP)
3759 | RT_BIT(X86_XCPT_DB)
3760 | RT_BIT(X86_XCPT_DE)
3761 | RT_BIT(X86_XCPT_NM)
3762 | RT_BIT(X86_XCPT_TS)
3763 | RT_BIT(X86_XCPT_UD)
3764 | RT_BIT(X86_XCPT_NP)
3765 | RT_BIT(X86_XCPT_SS)
3766 | RT_BIT(X86_XCPT_GP)
3767 | RT_BIT(X86_XCPT_PF)
3768 | RT_BIT(X86_XCPT_MF)
3769 ;
3770#elif defined(HMVMX_ALWAYS_TRAP_PF)
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3772#endif
3773
3774 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3775
3776 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3777 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3778 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3779 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3780 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3781 else
3782 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3783
3784 u32GuestCR0 |= uSetCR0;
3785 u32GuestCR0 &= uZapCR0;
3786 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3787
3788 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3789 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3790 AssertRCReturn(rc, rc);
3791 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3792 AssertRCReturn(rc, rc);
3793 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3794
3795 /*
3796 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3797 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3798 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3799 */
3800 uint32_t u32CR0Mask = 0;
3801 u32CR0Mask = X86_CR0_PE
3802 | X86_CR0_NE
3803 | X86_CR0_WP
3804 | X86_CR0_PG
3805 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3806 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3807 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3808
3809 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3810 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3811 * and @bugref{6944}. */
3812#if 0
3813 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3814 u32CR0Mask &= ~X86_CR0_PE;
3815#endif
3816 if (pVM->hm.s.fNestedPaging)
3817 u32CR0Mask &= ~X86_CR0_WP;
3818
3819 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3820 if (fInterceptNM)
3821 {
3822 u32CR0Mask |= X86_CR0_TS
3823 | X86_CR0_MP;
3824 }
3825
3826 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3827 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3828 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3829 AssertRCReturn(rc, rc);
3830 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3831
3832 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3833 }
3834 return rc;
3835}
3836
3837
3838/**
3839 * Loads the guest control registers (CR3, CR4) into the guest-state area
3840 * in the VMCS.
3841 *
3842 * @returns VBox status code.
3843 * @param pVM Pointer to the VM.
3844 * @param pVCpu Pointer to the VMCPU.
3845 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3846 * out-of-sync. Make sure to update the required fields
3847 * before using them.
3848 *
3849 * @remarks No-long-jump zone!!!
3850 */
3851static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3852{
3853 int rc = VINF_SUCCESS;
3854 PVM pVM = pVCpu->CTX_SUFF(pVM);
3855
3856 /*
3857 * Guest CR2.
3858 * It's always loaded in the assembler code. Nothing to do here.
3859 */
3860
3861 /*
3862 * Guest CR3.
3863 */
3864 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3865 {
3866 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3867 if (pVM->hm.s.fNestedPaging)
3868 {
3869 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3870
3871 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3872 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3873 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3874 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3875
3876 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3877 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3878 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3879
3880 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3881 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3882 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3883 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3884
3885 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3886 AssertRCReturn(rc, rc);
3887 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3888
3889 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3890 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3891 {
3892 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3893 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3894 {
3895 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3896 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3897 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3898 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3899 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3900 }
3901
3902 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3903 have Unrestricted Execution to handle the guest when it's not using paging. */
3904 GCPhysGuestCR3 = pMixedCtx->cr3;
3905 }
3906 else
3907 {
3908 /*
3909 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3910 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3911 * EPT takes care of translating it to host-physical addresses.
3912 */
3913 RTGCPHYS GCPhys;
3914 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3915 Assert(PDMVmmDevHeapIsEnabled(pVM));
3916
3917 /* We obtain it here every time as the guest could have relocated this PCI region. */
3918 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3919 AssertRCReturn(rc, rc);
3920
3921 GCPhysGuestCR3 = GCPhys;
3922 }
3923
3924 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3925 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3926 }
3927 else
3928 {
3929 /* Non-nested paging case, just use the hypervisor's CR3. */
3930 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3931
3932 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3933 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3934 }
3935 AssertRCReturn(rc, rc);
3936
3937 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3938 }
3939
3940 /*
3941 * Guest CR4.
3942 */
3943 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3944 {
3945 Assert(!(pMixedCtx->cr4 >> 32));
3946 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3947
3948 /* The guest's view of its CR4 is unblemished. */
3949 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3950 AssertRCReturn(rc, rc);
3951 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3952
3953 /* Setup VT-x's view of the guest CR4. */
3954 /*
3955 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3956 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3957 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3958 */
3959 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3960 {
3961 Assert(pVM->hm.s.vmx.pRealModeTSS);
3962 Assert(PDMVmmDevHeapIsEnabled(pVM));
3963 u32GuestCR4 &= ~X86_CR4_VME;
3964 }
3965
3966 if (pVM->hm.s.fNestedPaging)
3967 {
3968 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3969 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3970 {
3971 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3972 u32GuestCR4 |= X86_CR4_PSE;
3973 /* Our identity mapping is a 32-bit page directory. */
3974 u32GuestCR4 &= ~X86_CR4_PAE;
3975 }
3976 /* else use guest CR4.*/
3977 }
3978 else
3979 {
3980 /*
3981 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3982 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3983 */
3984 switch (pVCpu->hm.s.enmShadowMode)
3985 {
3986 case PGMMODE_REAL: /* Real-mode. */
3987 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3988 case PGMMODE_32_BIT: /* 32-bit paging. */
3989 {
3990 u32GuestCR4 &= ~X86_CR4_PAE;
3991 break;
3992 }
3993
3994 case PGMMODE_PAE: /* PAE paging. */
3995 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3996 {
3997 u32GuestCR4 |= X86_CR4_PAE;
3998 break;
3999 }
4000
4001 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4002 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4003#ifdef VBOX_ENABLE_64_BITS_GUESTS
4004 break;
4005#endif
4006 default:
4007 AssertFailed();
4008 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4009 }
4010 }
4011
4012 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4013 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4014 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4015 u32GuestCR4 |= uSetCR4;
4016 u32GuestCR4 &= uZapCR4;
4017
4018 /* Write VT-x's view of the guest CR4 into the VMCS. */
4019 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
4020 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4021 AssertRCReturn(rc, rc);
4022
4023 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4024 uint32_t u32CR4Mask = 0;
4025 u32CR4Mask = X86_CR4_VME
4026 | X86_CR4_PAE
4027 | X86_CR4_PGE
4028 | X86_CR4_PSE
4029 | X86_CR4_VMXE;
4030 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4031 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4032 AssertRCReturn(rc, rc);
4033
4034 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4035 }
4036 return rc;
4037}
4038
4039
4040/**
4041 * Loads the guest debug registers into the guest-state area in the VMCS.
4042 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4043 *
4044 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4045 *
4046 * @returns VBox status code.
4047 * @param pVCpu Pointer to the VMCPU.
4048 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4049 * out-of-sync. Make sure to update the required fields
4050 * before using them.
4051 *
4052 * @remarks No-long-jump zone!!!
4053 */
4054static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4055{
4056 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4057 return VINF_SUCCESS;
4058
4059#ifdef VBOX_STRICT
4060 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4061 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4062 {
4063 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4064 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4065 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4066 }
4067#endif
4068
4069 int rc;
4070 PVM pVM = pVCpu->CTX_SUFF(pVM);
4071 bool fInterceptDB = false;
4072 bool fInterceptMovDRx = false;
4073 if ( pVCpu->hm.s.fSingleInstruction
4074 || DBGFIsStepping(pVCpu))
4075 {
4076 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4077 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4078 {
4079 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4080 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4081 AssertRCReturn(rc, rc);
4082 Assert(fInterceptDB == false);
4083 }
4084 else
4085 {
4086 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4087 pVCpu->hm.s.fClearTrapFlag = true;
4088 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4089 fInterceptDB = true;
4090 }
4091 }
4092
4093 if ( fInterceptDB
4094 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4095 {
4096 /*
4097 * Use the combined guest and host DRx values found in the hypervisor
4098 * register set because the debugger has breakpoints active or someone
4099 * is single stepping on the host side without a monitor trap flag.
4100 *
4101 * Note! DBGF expects a clean DR6 state before executing guest code.
4102 */
4103#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4104 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4105 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4106 {
4107 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4108 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4109 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4110 }
4111 else
4112#endif
4113 if (!CPUMIsHyperDebugStateActive(pVCpu))
4114 {
4115 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4116 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4117 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4118 }
4119
4120 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4121 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4122 AssertRCReturn(rc, rc);
4123
4124 pVCpu->hm.s.fUsingHyperDR7 = true;
4125 fInterceptDB = true;
4126 fInterceptMovDRx = true;
4127 }
4128 else
4129 {
4130 /*
4131 * If the guest has enabled debug registers, we need to load them prior to
4132 * executing guest code so they'll trigger at the right time.
4133 */
4134 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4135 {
4136#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4137 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4138 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4139 {
4140 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4141 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4142 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4143 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4144 }
4145 else
4146#endif
4147 if (!CPUMIsGuestDebugStateActive(pVCpu))
4148 {
4149 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4150 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4151 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4152 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4153 }
4154 Assert(!fInterceptDB);
4155 Assert(!fInterceptMovDRx);
4156 }
4157 /*
4158 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4159 * must intercept #DB in order to maintain a correct DR6 guest value.
4160 */
4161#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4162 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4163 && !CPUMIsGuestDebugStateActive(pVCpu))
4164#else
4165 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4166#endif
4167 {
4168 fInterceptMovDRx = true;
4169 fInterceptDB = true;
4170 }
4171
4172 /* Update guest DR7. */
4173 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4174 AssertRCReturn(rc, rc);
4175
4176 pVCpu->hm.s.fUsingHyperDR7 = false;
4177 }
4178
4179 /*
4180 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4181 */
4182 if (fInterceptDB)
4183 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4184 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4185 {
4186#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4187 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4188#endif
4189 }
4190 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4191 AssertRCReturn(rc, rc);
4192
4193 /*
4194 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4195 */
4196 if (fInterceptMovDRx)
4197 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4198 else
4199 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4200 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4201 AssertRCReturn(rc, rc);
4202
4203 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4204 return VINF_SUCCESS;
4205}
4206
4207
4208#ifdef VBOX_STRICT
4209/**
4210 * Strict function to validate segment registers.
4211 *
4212 * @remarks ASSUMES CR0 is up to date.
4213 */
4214static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4215{
4216 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4217 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4218 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4219 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4220 && ( !CPUMIsGuestInRealModeEx(pCtx)
4221 && !CPUMIsGuestInV86ModeEx(pCtx)))
4222 {
4223 /* Protected mode checks */
4224 /* CS */
4225 Assert(pCtx->cs.Attr.n.u1Present);
4226 Assert(!(pCtx->cs.Attr.u & 0xf00));
4227 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4228 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4229 || !(pCtx->cs.Attr.n.u1Granularity));
4230 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4231 || (pCtx->cs.Attr.n.u1Granularity));
4232 /* CS cannot be loaded with NULL in protected mode. */
4233 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4234 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4235 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4236 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4237 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4238 else
4239 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4240 /* SS */
4241 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4242 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4243 if ( !(pCtx->cr0 & X86_CR0_PE)
4244 || pCtx->cs.Attr.n.u4Type == 3)
4245 {
4246 Assert(!pCtx->ss.Attr.n.u2Dpl);
4247 }
4248 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4249 {
4250 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4251 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4252 Assert(pCtx->ss.Attr.n.u1Present);
4253 Assert(!(pCtx->ss.Attr.u & 0xf00));
4254 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4255 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4256 || !(pCtx->ss.Attr.n.u1Granularity));
4257 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4258 || (pCtx->ss.Attr.n.u1Granularity));
4259 }
4260 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4261 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4262 {
4263 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4264 Assert(pCtx->ds.Attr.n.u1Present);
4265 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4266 Assert(!(pCtx->ds.Attr.u & 0xf00));
4267 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4268 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4269 || !(pCtx->ds.Attr.n.u1Granularity));
4270 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4271 || (pCtx->ds.Attr.n.u1Granularity));
4272 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4273 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4274 }
4275 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4276 {
4277 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4278 Assert(pCtx->es.Attr.n.u1Present);
4279 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4280 Assert(!(pCtx->es.Attr.u & 0xf00));
4281 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4282 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4283 || !(pCtx->es.Attr.n.u1Granularity));
4284 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4285 || (pCtx->es.Attr.n.u1Granularity));
4286 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4287 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4288 }
4289 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4290 {
4291 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4292 Assert(pCtx->fs.Attr.n.u1Present);
4293 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4294 Assert(!(pCtx->fs.Attr.u & 0xf00));
4295 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4296 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4297 || !(pCtx->fs.Attr.n.u1Granularity));
4298 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4299 || (pCtx->fs.Attr.n.u1Granularity));
4300 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4301 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4302 }
4303 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4304 {
4305 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4306 Assert(pCtx->gs.Attr.n.u1Present);
4307 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4308 Assert(!(pCtx->gs.Attr.u & 0xf00));
4309 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4310 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4311 || !(pCtx->gs.Attr.n.u1Granularity));
4312 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4313 || (pCtx->gs.Attr.n.u1Granularity));
4314 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4315 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4316 }
4317 /* 64-bit capable CPUs. */
4318# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4319 if (HMVMX_IS_64BIT_HOST_MODE())
4320 {
4321 Assert(!(pCtx->cs.u64Base >> 32));
4322 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4323 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4324 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4325 }
4326# endif
4327 }
4328 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4329 || ( CPUMIsGuestInRealModeEx(pCtx)
4330 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4331 {
4332 /* Real and v86 mode checks. */
4333 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4334 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4335 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4336 {
4337 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4338 }
4339 else
4340 {
4341 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4342 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4343 }
4344
4345 /* CS */
4346 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4347 Assert(pCtx->cs.u32Limit == 0xffff);
4348 Assert(u32CSAttr == 0xf3);
4349 /* SS */
4350 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4351 Assert(pCtx->ss.u32Limit == 0xffff);
4352 Assert(u32SSAttr == 0xf3);
4353 /* DS */
4354 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4355 Assert(pCtx->ds.u32Limit == 0xffff);
4356 Assert(u32DSAttr == 0xf3);
4357 /* ES */
4358 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4359 Assert(pCtx->es.u32Limit == 0xffff);
4360 Assert(u32ESAttr == 0xf3);
4361 /* FS */
4362 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4363 Assert(pCtx->fs.u32Limit == 0xffff);
4364 Assert(u32FSAttr == 0xf3);
4365 /* GS */
4366 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4367 Assert(pCtx->gs.u32Limit == 0xffff);
4368 Assert(u32GSAttr == 0xf3);
4369 /* 64-bit capable CPUs. */
4370# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4371 if (HMVMX_IS_64BIT_HOST_MODE())
4372 {
4373 Assert(!(pCtx->cs.u64Base >> 32));
4374 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4375 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4376 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4377 }
4378# endif
4379 }
4380}
4381#endif /* VBOX_STRICT */
4382
4383
4384/**
4385 * Writes a guest segment register into the guest-state area in the VMCS.
4386 *
4387 * @returns VBox status code.
4388 * @param pVCpu Pointer to the VMCPU.
4389 * @param idxSel Index of the selector in the VMCS.
4390 * @param idxLimit Index of the segment limit in the VMCS.
4391 * @param idxBase Index of the segment base in the VMCS.
4392 * @param idxAccess Index of the access rights of the segment in the VMCS.
4393 * @param pSelReg Pointer to the segment selector.
4394 *
4395 * @remarks No-long-jump zone!!!
4396 */
4397static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4398 uint32_t idxAccess, PCPUMSELREG pSelReg)
4399{
4400 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4401 AssertRCReturn(rc, rc);
4402 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4403 AssertRCReturn(rc, rc);
4404 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4405 AssertRCReturn(rc, rc);
4406
4407 uint32_t u32Access = pSelReg->Attr.u;
4408 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4409 {
4410 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4411 u32Access = 0xf3;
4412 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4413 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4414 }
4415 else
4416 {
4417 /*
4418 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4419 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4420 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4421 * loaded in protected-mode have their attribute as 0.
4422 */
4423 if (!u32Access)
4424 u32Access = X86DESCATTR_UNUSABLE;
4425 }
4426
4427 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4428 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4429 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4430
4431 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4432 AssertRCReturn(rc, rc);
4433 return rc;
4434}
4435
4436
4437/**
4438 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4439 * into the guest-state area in the VMCS.
4440 *
4441 * @returns VBox status code.
4442 * @param pVM Pointer to the VM.
4443 * @param pVCPU Pointer to the VMCPU.
4444 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4445 * out-of-sync. Make sure to update the required fields
4446 * before using them.
4447 *
4448 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4449 * @remarks No-long-jump zone!!!
4450 */
4451static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4452{
4453 int rc = VERR_INTERNAL_ERROR_5;
4454 PVM pVM = pVCpu->CTX_SUFF(pVM);
4455
4456 /*
4457 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4458 */
4459 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4460 {
4461 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4462 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4463 {
4464 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4465 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4466 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4467 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4468 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4469 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4470 }
4471
4472#ifdef VBOX_WITH_REM
4473 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4474 {
4475 Assert(pVM->hm.s.vmx.pRealModeTSS);
4476 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4477 if ( pVCpu->hm.s.vmx.fWasInRealMode
4478 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4479 {
4480 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4481 in real-mode (e.g. OpenBSD 4.0) */
4482 REMFlushTBs(pVM);
4483 Log4(("Load: Switch to protected mode detected!\n"));
4484 pVCpu->hm.s.vmx.fWasInRealMode = false;
4485 }
4486 }
4487#endif
4488 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4489 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4490 AssertRCReturn(rc, rc);
4491 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4492 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4493 AssertRCReturn(rc, rc);
4494 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4495 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4496 AssertRCReturn(rc, rc);
4497 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4498 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4499 AssertRCReturn(rc, rc);
4500 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4501 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4502 AssertRCReturn(rc, rc);
4503 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4504 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4505 AssertRCReturn(rc, rc);
4506
4507#ifdef VBOX_STRICT
4508 /* Validate. */
4509 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4510#endif
4511
4512 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4513 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4514 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4515 }
4516
4517 /*
4518 * Guest TR.
4519 */
4520 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4521 {
4522 /*
4523 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4524 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4525 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4526 */
4527 uint16_t u16Sel = 0;
4528 uint32_t u32Limit = 0;
4529 uint64_t u64Base = 0;
4530 uint32_t u32AccessRights = 0;
4531
4532 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4533 {
4534 u16Sel = pMixedCtx->tr.Sel;
4535 u32Limit = pMixedCtx->tr.u32Limit;
4536 u64Base = pMixedCtx->tr.u64Base;
4537 u32AccessRights = pMixedCtx->tr.Attr.u;
4538 }
4539 else
4540 {
4541 Assert(pVM->hm.s.vmx.pRealModeTSS);
4542 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4543
4544 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4545 RTGCPHYS GCPhys;
4546 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4547 AssertRCReturn(rc, rc);
4548
4549 X86DESCATTR DescAttr;
4550 DescAttr.u = 0;
4551 DescAttr.n.u1Present = 1;
4552 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4553
4554 u16Sel = 0;
4555 u32Limit = HM_VTX_TSS_SIZE;
4556 u64Base = GCPhys; /* in real-mode phys = virt. */
4557 u32AccessRights = DescAttr.u;
4558 }
4559
4560 /* Validate. */
4561 Assert(!(u16Sel & RT_BIT(2)));
4562 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4563 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4564 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4565 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4566 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4567 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4568 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4569 Assert( (u32Limit & 0xfff) == 0xfff
4570 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4571 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4572 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4573
4574 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4575 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4576 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4577 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4578
4579 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4580 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4581 }
4582
4583 /*
4584 * Guest GDTR.
4585 */
4586 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4587 {
4588 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4589 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4590
4591 /* Validate. */
4592 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4593
4594 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4595 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4596 }
4597
4598 /*
4599 * Guest LDTR.
4600 */
4601 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4602 {
4603 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4604 uint32_t u32Access = 0;
4605 if (!pMixedCtx->ldtr.Attr.u)
4606 u32Access = X86DESCATTR_UNUSABLE;
4607 else
4608 u32Access = pMixedCtx->ldtr.Attr.u;
4609
4610 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4611 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4612 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4613 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4614
4615 /* Validate. */
4616 if (!(u32Access & X86DESCATTR_UNUSABLE))
4617 {
4618 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4619 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4620 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4621 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4622 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4623 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4624 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4625 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4626 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4627 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4628 }
4629
4630 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4631 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4632 }
4633
4634 /*
4635 * Guest IDTR.
4636 */
4637 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4638 {
4639 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4640 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4641
4642 /* Validate. */
4643 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4644
4645 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4646 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4647 }
4648
4649 return VINF_SUCCESS;
4650}
4651
4652
4653/**
4654 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4655 * areas. These MSRs will automatically be loaded to the host CPU on every
4656 * successful VM entry and stored from the host CPU on every successful VM-exit.
4657 *
4658 * This also creates/updates MSR slots for the host MSRs. The actual host
4659 * MSR values are -not- updated here for performance reasons. See
4660 * hmR0VmxSaveHostMsrs().
4661 *
4662 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4663 *
4664 * @returns VBox status code.
4665 * @param pVCpu Pointer to the VMCPU.
4666 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4667 * out-of-sync. Make sure to update the required fields
4668 * before using them.
4669 *
4670 * @remarks No-long-jump zone!!!
4671 */
4672static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4673{
4674 AssertPtr(pVCpu);
4675 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4676
4677 /*
4678 * MSRs that we use the auto-load/store MSR area in the VMCS.
4679 */
4680 PVM pVM = pVCpu->CTX_SUFF(pVM);
4681 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4682 {
4683 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4684#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4685 if (pVM->hm.s.fAllow64BitGuests)
4686 {
4687 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4688 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4689 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4690 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4691# ifdef DEBUG
4692 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4693 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4694 Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4695# endif
4696 }
4697#endif
4698 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4699 }
4700
4701 /*
4702 * Guest Sysenter MSRs.
4703 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4704 * VM-exits on WRMSRs for these MSRs.
4705 */
4706 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4707 {
4708 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4709 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4710 }
4711
4712 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4713 {
4714 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4715 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4716 }
4717
4718 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4719 {
4720 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4721 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4722 }
4723
4724 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4725 {
4726 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4727 {
4728 /*
4729 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4730 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4731 */
4732 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4733 {
4734 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4735 AssertRCReturn(rc,rc);
4736 Log4(("Load: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pMixedCtx->msrEFER));
4737 }
4738 else
4739 {
4740 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4741 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4742 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4743 Log4(("Load: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4744 pVCpu->hm.s.vmx.cMsrs));
4745 }
4746 }
4747 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4748 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4749 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4750 }
4751
4752 return VINF_SUCCESS;
4753}
4754
4755
4756/**
4757 * Loads the guest activity state into the guest-state area in the VMCS.
4758 *
4759 * @returns VBox status code.
4760 * @param pVCpu Pointer to the VMCPU.
4761 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4762 * out-of-sync. Make sure to update the required fields
4763 * before using them.
4764 *
4765 * @remarks No-long-jump zone!!!
4766 */
4767static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4768{
4769 NOREF(pCtx);
4770 /** @todo See if we can make use of other states, e.g.
4771 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4772 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4773 {
4774 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4775 AssertRCReturn(rc, rc);
4776
4777 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4778 }
4779 return VINF_SUCCESS;
4780}
4781
4782
4783/**
4784 * Sets up the appropriate function to run guest code.
4785 *
4786 * @returns VBox status code.
4787 * @param pVCpu Pointer to the VMCPU.
4788 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4789 * out-of-sync. Make sure to update the required fields
4790 * before using them.
4791 *
4792 * @remarks No-long-jump zone!!!
4793 */
4794static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4795{
4796 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4797 {
4798#ifndef VBOX_ENABLE_64_BITS_GUESTS
4799 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4800#endif
4801 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4802#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4803 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4804 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4805 {
4806 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4807 {
4808 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4809 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4810 | HM_CHANGED_VMX_EXIT_CTLS
4811 | HM_CHANGED_VMX_ENTRY_CTLS
4812 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4813 }
4814 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4815 }
4816#else
4817 /* 64-bit host or hybrid host. */
4818 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4819#endif
4820 }
4821 else
4822 {
4823 /* Guest is not in long mode, use the 32-bit handler. */
4824#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4825 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4826 {
4827 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4828 {
4829 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4830 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4831 | HM_CHANGED_VMX_EXIT_CTLS
4832 | HM_CHANGED_VMX_ENTRY_CTLS
4833 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4834 }
4835 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4836 }
4837#else
4838 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4839#endif
4840 }
4841 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4842 return VINF_SUCCESS;
4843}
4844
4845
4846/**
4847 * Wrapper for running the guest code in VT-x.
4848 *
4849 * @returns VBox strict status code.
4850 * @param pVM Pointer to the VM.
4851 * @param pVCpu Pointer to the VMCPU.
4852 * @param pCtx Pointer to the guest-CPU context.
4853 *
4854 * @remarks No-long-jump zone!!!
4855 */
4856DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4857{
4858 /*
4859 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4860 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4861 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4862 */
4863 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4864 /** @todo Add stats for resume vs launch. */
4865#ifdef VBOX_WITH_KERNEL_USING_XMM
4866 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4867#else
4868 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4869#endif
4870}
4871
4872
4873/**
4874 * Reports world-switch error and dumps some useful debug info.
4875 *
4876 * @param pVM Pointer to the VM.
4877 * @param pVCpu Pointer to the VMCPU.
4878 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4879 * @param pCtx Pointer to the guest-CPU context.
4880 * @param pVmxTransient Pointer to the VMX transient structure (only
4881 * exitReason updated).
4882 */
4883static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4884{
4885 Assert(pVM);
4886 Assert(pVCpu);
4887 Assert(pCtx);
4888 Assert(pVmxTransient);
4889 HMVMX_ASSERT_PREEMPT_SAFE();
4890
4891 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4892 switch (rcVMRun)
4893 {
4894 case VERR_VMX_INVALID_VMXON_PTR:
4895 AssertFailed();
4896 break;
4897 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4898 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4899 {
4900 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4901 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4902 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4903 AssertRC(rc);
4904
4905 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4906 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4907 Cannot do it here as we may have been long preempted. */
4908
4909#ifdef VBOX_STRICT
4910 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4911 pVmxTransient->uExitReason));
4912 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4913 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4914 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4915 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4916 else
4917 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4918 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4919 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4920
4921 /* VMX control bits. */
4922 uint32_t u32Val;
4923 uint64_t u64Val;
4924 HMVMXHCUINTREG uHCReg;
4925 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4926 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4927 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4928 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4929 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4930 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4931 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4932 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4933 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4934 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4935 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4936 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4937 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4938 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4939 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4940 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4941 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4942 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4943 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4944 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4945 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4946 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4947 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4948 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4949 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4950 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4951 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4952 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4953 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4954 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4955 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4956 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4957 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4958 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4959 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4960 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4961 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4962 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4963 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4964 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4965 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4966 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4967
4968 /* Guest bits. */
4969 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4970 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4971 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4972 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4973 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4974 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4975 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4976 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4977
4978 /* Host bits. */
4979 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4980 Log4(("Host CR0 %#RHr\n", uHCReg));
4981 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4982 Log4(("Host CR3 %#RHr\n", uHCReg));
4983 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4984 Log4(("Host CR4 %#RHr\n", uHCReg));
4985
4986 RTGDTR HostGdtr;
4987 PCX86DESCHC pDesc;
4988 ASMGetGDTR(&HostGdtr);
4989 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4990 Log4(("Host CS %#08x\n", u32Val));
4991 if (u32Val < HostGdtr.cbGdt)
4992 {
4993 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4994 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4995 }
4996
4997 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4998 Log4(("Host DS %#08x\n", u32Val));
4999 if (u32Val < HostGdtr.cbGdt)
5000 {
5001 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5002 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5003 }
5004
5005 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5006 Log4(("Host ES %#08x\n", u32Val));
5007 if (u32Val < HostGdtr.cbGdt)
5008 {
5009 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5010 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5011 }
5012
5013 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5014 Log4(("Host FS %#08x\n", u32Val));
5015 if (u32Val < HostGdtr.cbGdt)
5016 {
5017 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5018 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5019 }
5020
5021 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5022 Log4(("Host GS %#08x\n", u32Val));
5023 if (u32Val < HostGdtr.cbGdt)
5024 {
5025 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5026 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5027 }
5028
5029 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5030 Log4(("Host SS %#08x\n", u32Val));
5031 if (u32Val < HostGdtr.cbGdt)
5032 {
5033 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5034 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5035 }
5036
5037 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5038 Log4(("Host TR %#08x\n", u32Val));
5039 if (u32Val < HostGdtr.cbGdt)
5040 {
5041 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5042 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5043 }
5044
5045 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5046 Log4(("Host TR Base %#RHv\n", uHCReg));
5047 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5048 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5049 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5050 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5051 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5052 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5053 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5054 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5055 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5056 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5057 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5058 Log4(("Host RSP %#RHv\n", uHCReg));
5059 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5060 Log4(("Host RIP %#RHv\n", uHCReg));
5061# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5062 if (HMVMX_IS_64BIT_HOST_MODE())
5063 {
5064 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5065 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5066 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5067 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5068 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5069 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5070 }
5071# endif
5072#endif /* VBOX_STRICT */
5073 break;
5074 }
5075
5076 default:
5077 /* Impossible */
5078 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5079 break;
5080 }
5081 NOREF(pVM); NOREF(pCtx);
5082}
5083
5084
5085#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5086#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5087# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5088#endif
5089#ifdef VBOX_STRICT
5090static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5091{
5092 switch (idxField)
5093 {
5094 case VMX_VMCS_GUEST_RIP:
5095 case VMX_VMCS_GUEST_RSP:
5096 case VMX_VMCS_GUEST_SYSENTER_EIP:
5097 case VMX_VMCS_GUEST_SYSENTER_ESP:
5098 case VMX_VMCS_GUEST_GDTR_BASE:
5099 case VMX_VMCS_GUEST_IDTR_BASE:
5100 case VMX_VMCS_GUEST_CS_BASE:
5101 case VMX_VMCS_GUEST_DS_BASE:
5102 case VMX_VMCS_GUEST_ES_BASE:
5103 case VMX_VMCS_GUEST_FS_BASE:
5104 case VMX_VMCS_GUEST_GS_BASE:
5105 case VMX_VMCS_GUEST_SS_BASE:
5106 case VMX_VMCS_GUEST_LDTR_BASE:
5107 case VMX_VMCS_GUEST_TR_BASE:
5108 case VMX_VMCS_GUEST_CR3:
5109 return true;
5110 }
5111 return false;
5112}
5113
5114static bool hmR0VmxIsValidReadField(uint32_t idxField)
5115{
5116 switch (idxField)
5117 {
5118 /* Read-only fields. */
5119 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5120 return true;
5121 }
5122 /* Remaining readable fields should also be writable. */
5123 return hmR0VmxIsValidWriteField(idxField);
5124}
5125#endif /* VBOX_STRICT */
5126
5127
5128/**
5129 * Executes the specified handler in 64-bit mode.
5130 *
5131 * @returns VBox status code.
5132 * @param pVM Pointer to the VM.
5133 * @param pVCpu Pointer to the VMCPU.
5134 * @param pCtx Pointer to the guest CPU context.
5135 * @param enmOp The operation to perform.
5136 * @param cbParam Number of parameters.
5137 * @param paParam Array of 32-bit parameters.
5138 */
5139VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5140 uint32_t *paParam)
5141{
5142 int rc, rc2;
5143 PHMGLOBALCPUINFO pCpu;
5144 RTHCPHYS HCPhysCpuPage;
5145 RTCCUINTREG uOldEflags;
5146
5147 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5148 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5149 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5150 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5151
5152#ifdef VBOX_STRICT
5153 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5154 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5155
5156 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5157 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5158#endif
5159
5160 /* Disable interrupts. */
5161 uOldEflags = ASMIntDisableFlags();
5162
5163#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5164 RTCPUID idHostCpu = RTMpCpuId();
5165 CPUMR0SetLApic(pVCpu, idHostCpu);
5166#endif
5167
5168 pCpu = HMR0GetCurrentCpu();
5169 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5170
5171 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5172 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5173
5174 /* Leave VMX Root Mode. */
5175 VMXDisable();
5176
5177 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5178
5179 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5180 CPUMSetHyperEIP(pVCpu, enmOp);
5181 for (int i = (int)cbParam - 1; i >= 0; i--)
5182 CPUMPushHyper(pVCpu, paParam[i]);
5183
5184 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5185
5186 /* Call the switcher. */
5187 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5188 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5189
5190 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5191 /* Make sure the VMX instructions don't cause #UD faults. */
5192 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5193
5194 /* Re-enter VMX Root Mode */
5195 rc2 = VMXEnable(HCPhysCpuPage);
5196 if (RT_FAILURE(rc2))
5197 {
5198 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5199 ASMSetFlags(uOldEflags);
5200 return rc2;
5201 }
5202
5203 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5204 AssertRC(rc2);
5205 Assert(!(ASMGetFlags() & X86_EFL_IF));
5206 ASMSetFlags(uOldEflags);
5207 return rc;
5208}
5209
5210
5211/**
5212 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5213 * supporting 64-bit guests.
5214 *
5215 * @returns VBox status code.
5216 * @param fResume Whether to VMLAUNCH or VMRESUME.
5217 * @param pCtx Pointer to the guest-CPU context.
5218 * @param pCache Pointer to the VMCS cache.
5219 * @param pVM Pointer to the VM.
5220 * @param pVCpu Pointer to the VMCPU.
5221 */
5222DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5223{
5224 uint32_t aParam[6];
5225 PHMGLOBALCPUINFO pCpu = NULL;
5226 RTHCPHYS HCPhysCpuPage = 0;
5227 int rc = VERR_INTERNAL_ERROR_5;
5228
5229 pCpu = HMR0GetCurrentCpu();
5230 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5231
5232#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5233 pCache->uPos = 1;
5234 pCache->interPD = PGMGetInterPaeCR3(pVM);
5235 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5236#endif
5237
5238#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5239 pCache->TestIn.HCPhysCpuPage = 0;
5240 pCache->TestIn.HCPhysVmcs = 0;
5241 pCache->TestIn.pCache = 0;
5242 pCache->TestOut.HCPhysVmcs = 0;
5243 pCache->TestOut.pCache = 0;
5244 pCache->TestOut.pCtx = 0;
5245 pCache->TestOut.eflags = 0;
5246#endif
5247
5248 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5249 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5250 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5251 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5252 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5253 aParam[5] = 0;
5254
5255#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5256 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5257 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5258#endif
5259 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5260
5261#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5262 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5263 Assert(pCtx->dr[4] == 10);
5264 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5265#endif
5266
5267#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5268 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5269 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5270 pVCpu->hm.s.vmx.HCPhysVmcs));
5271 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5272 pCache->TestOut.HCPhysVmcs));
5273 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5274 pCache->TestOut.pCache));
5275 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5276 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5277 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5278 pCache->TestOut.pCtx));
5279 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5280#endif
5281 return rc;
5282}
5283
5284
5285/**
5286 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5287 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5288 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5289 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5290 *
5291 * @returns VBox status code.
5292 * @param pVM Pointer to the VM.
5293 * @param pVCpu Pointer to the VMCPU.
5294 */
5295static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5296{
5297#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5298{ \
5299 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5300 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5301 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5302 ++cReadFields; \
5303}
5304
5305 AssertPtr(pVM);
5306 AssertPtr(pVCpu);
5307 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5308 uint32_t cReadFields = 0;
5309
5310 /*
5311 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5312 * and serve to indicate exceptions to the rules.
5313 */
5314
5315 /* Guest-natural selector base fields. */
5316#if 0
5317 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5318 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5319 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5320#endif
5321 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5322 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5323 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5324 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5325 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5326 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5327 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5328 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5329 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5330 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5331 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5332 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5333#if 0
5334 /* Unused natural width guest-state fields. */
5335 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5336 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5337#endif
5338 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5339 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5340
5341 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5342#if 0
5343 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5344 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5345 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5346 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5347 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5348 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5349 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5350 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5351 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5352#endif
5353
5354 /* Natural width guest-state fields. */
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5356#if 0
5357 /* Currently unused field. */
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5359#endif
5360
5361 if (pVM->hm.s.fNestedPaging)
5362 {
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5364 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5365 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5366 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5367 }
5368 else
5369 {
5370 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5371 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5372 }
5373
5374#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5375 return VINF_SUCCESS;
5376}
5377
5378
5379/**
5380 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5381 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5382 * darwin, running 64-bit guests).
5383 *
5384 * @returns VBox status code.
5385 * @param pVCpu Pointer to the VMCPU.
5386 * @param idxField The VMCS field encoding.
5387 * @param u64Val 16, 32 or 64-bit value.
5388 */
5389VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5390{
5391 int rc;
5392 switch (idxField)
5393 {
5394 /*
5395 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5396 */
5397 /* 64-bit Control fields. */
5398 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5399 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5400 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5401 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5402 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5403 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5404 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5405 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5406 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5407 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5408 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5409 case VMX_VMCS64_CTRL_EPTP_FULL:
5410 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5411 /* 64-bit Guest-state fields. */
5412 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5413 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5414 case VMX_VMCS64_GUEST_PAT_FULL:
5415 case VMX_VMCS64_GUEST_EFER_FULL:
5416 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5417 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5418 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5419 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5420 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5421 /* 64-bit Host-state fields. */
5422 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5423 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5424 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5425 {
5426 rc = VMXWriteVmcs32(idxField, u64Val);
5427 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5428 break;
5429 }
5430
5431 /*
5432 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5433 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5434 */
5435 /* Natural-width Guest-state fields. */
5436 case VMX_VMCS_GUEST_CR3:
5437 case VMX_VMCS_GUEST_ES_BASE:
5438 case VMX_VMCS_GUEST_CS_BASE:
5439 case VMX_VMCS_GUEST_SS_BASE:
5440 case VMX_VMCS_GUEST_DS_BASE:
5441 case VMX_VMCS_GUEST_FS_BASE:
5442 case VMX_VMCS_GUEST_GS_BASE:
5443 case VMX_VMCS_GUEST_LDTR_BASE:
5444 case VMX_VMCS_GUEST_TR_BASE:
5445 case VMX_VMCS_GUEST_GDTR_BASE:
5446 case VMX_VMCS_GUEST_IDTR_BASE:
5447 case VMX_VMCS_GUEST_RSP:
5448 case VMX_VMCS_GUEST_RIP:
5449 case VMX_VMCS_GUEST_SYSENTER_ESP:
5450 case VMX_VMCS_GUEST_SYSENTER_EIP:
5451 {
5452 if (!(u64Val >> 32))
5453 {
5454 /* If this field is 64-bit, VT-x will zero out the top bits. */
5455 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5456 }
5457 else
5458 {
5459 /* Assert that only the 32->64 switcher case should ever come here. */
5460 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5461 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5462 }
5463 break;
5464 }
5465
5466 default:
5467 {
5468 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5469 rc = VERR_INVALID_PARAMETER;
5470 break;
5471 }
5472 }
5473 AssertRCReturn(rc, rc);
5474 return rc;
5475}
5476
5477
5478/**
5479 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5480 * hosts (except darwin) for 64-bit guests.
5481 *
5482 * @param pVCpu Pointer to the VMCPU.
5483 * @param idxField The VMCS field encoding.
5484 * @param u64Val 16, 32 or 64-bit value.
5485 */
5486VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5487{
5488 AssertPtr(pVCpu);
5489 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5490
5491 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5492 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5493
5494 /* Make sure there are no duplicates. */
5495 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5496 {
5497 if (pCache->Write.aField[i] == idxField)
5498 {
5499 pCache->Write.aFieldVal[i] = u64Val;
5500 return VINF_SUCCESS;
5501 }
5502 }
5503
5504 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5505 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5506 pCache->Write.cValidEntries++;
5507 return VINF_SUCCESS;
5508}
5509
5510/* Enable later when the assembly code uses these as callbacks. */
5511#if 0
5512/*
5513 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5514 *
5515 * @param pVCpu Pointer to the VMCPU.
5516 * @param pCache Pointer to the VMCS cache.
5517 *
5518 * @remarks No-long-jump zone!!!
5519 */
5520VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5521{
5522 AssertPtr(pCache);
5523 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5524 {
5525 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5526 AssertRC(rc);
5527 }
5528 pCache->Write.cValidEntries = 0;
5529}
5530
5531
5532/**
5533 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5534 *
5535 * @param pVCpu Pointer to the VMCPU.
5536 * @param pCache Pointer to the VMCS cache.
5537 *
5538 * @remarks No-long-jump zone!!!
5539 */
5540VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5541{
5542 AssertPtr(pCache);
5543 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5544 {
5545 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5546 AssertRC(rc);
5547 }
5548}
5549#endif
5550#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5551
5552
5553/**
5554 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5555 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5556 * timer.
5557 *
5558 * @returns VBox status code.
5559 * @param pVCpu Pointer to the VMCPU.
5560 *
5561 * @remarks No-long-jump zone!!!
5562 */
5563static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5564{
5565 int rc = VERR_INTERNAL_ERROR_5;
5566 bool fOffsettedTsc = false;
5567 PVM pVM = pVCpu->CTX_SUFF(pVM);
5568 if (pVM->hm.s.vmx.fUsePreemptTimer)
5569 {
5570 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5571
5572 /* Make sure the returned values have sane upper and lower boundaries. */
5573 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5574 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5575 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5576 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5577
5578 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5579 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5580 }
5581 else
5582 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5583
5584 if (fOffsettedTsc)
5585 {
5586 uint64_t u64CurTSC = ASMReadTSC();
5587 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5588 {
5589 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5590 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5591
5592 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5593 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5594 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5595 }
5596 else
5597 {
5598 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5599 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5600 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5601 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5602 }
5603 }
5604 else
5605 {
5606 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5607 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5608 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5609 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5610 }
5611}
5612
5613
5614/**
5615 * Determines if an exception is a contributory exception. Contributory
5616 * exceptions are ones which can cause double-faults. Page-fault is
5617 * intentionally not included here as it's a conditional contributory exception.
5618 *
5619 * @returns true if the exception is contributory, false otherwise.
5620 * @param uVector The exception vector.
5621 */
5622DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5623{
5624 switch (uVector)
5625 {
5626 case X86_XCPT_GP:
5627 case X86_XCPT_SS:
5628 case X86_XCPT_NP:
5629 case X86_XCPT_TS:
5630 case X86_XCPT_DE:
5631 return true;
5632 default:
5633 break;
5634 }
5635 return false;
5636}
5637
5638
5639/**
5640 * Sets an event as a pending event to be injected into the guest.
5641 *
5642 * @param pVCpu Pointer to the VMCPU.
5643 * @param u32IntInfo The VM-entry interruption-information field.
5644 * @param cbInstr The VM-entry instruction length in bytes (for software
5645 * interrupts, exceptions and privileged software
5646 * exceptions).
5647 * @param u32ErrCode The VM-entry exception error code.
5648 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5649 * page-fault.
5650 *
5651 * @remarks Statistics counter assumes this is a guest event being injected or
5652 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5653 * always incremented.
5654 */
5655DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5656 RTGCUINTPTR GCPtrFaultAddress)
5657{
5658 Assert(!pVCpu->hm.s.Event.fPending);
5659 pVCpu->hm.s.Event.fPending = true;
5660 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5661 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5662 pVCpu->hm.s.Event.cbInstr = cbInstr;
5663 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5664
5665 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5666}
5667
5668
5669/**
5670 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5671 *
5672 * @param pVCpu Pointer to the VMCPU.
5673 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5674 * out-of-sync. Make sure to update the required fields
5675 * before using them.
5676 */
5677DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5678{
5679 NOREF(pMixedCtx);
5680 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5681 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5682 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5683 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5684}
5685
5686
5687/**
5688 * Handle a condition that occurred while delivering an event through the guest
5689 * IDT.
5690 *
5691 * @returns VBox status code (informational error codes included).
5692 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5693 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5694 * continue execution of the guest which will delivery the #DF.
5695 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5696 *
5697 * @param pVCpu Pointer to the VMCPU.
5698 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5699 * out-of-sync. Make sure to update the required fields
5700 * before using them.
5701 * @param pVmxTransient Pointer to the VMX transient structure.
5702 *
5703 * @remarks No-long-jump zone!!!
5704 */
5705static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5706{
5707 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5708 AssertRCReturn(rc, rc);
5709 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5710 {
5711 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5712 AssertRCReturn(rc, rc);
5713
5714 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5715 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5716 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5717
5718 typedef enum
5719 {
5720 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5721 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5722 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5723 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5724 } VMXREFLECTXCPT;
5725
5726 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5727 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5728 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5729 {
5730 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5731 {
5732 enmReflect = VMXREFLECTXCPT_XCPT;
5733#ifdef VBOX_STRICT
5734 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5735 && uExitVector == X86_XCPT_PF)
5736 {
5737 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5738 }
5739#endif
5740 if ( uExitVector == X86_XCPT_PF
5741 && uIdtVector == X86_XCPT_PF)
5742 {
5743 pVmxTransient->fVectoringPF = true;
5744 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5745 }
5746 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5747 && hmR0VmxIsContributoryXcpt(uExitVector)
5748 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5749 || uIdtVector == X86_XCPT_PF))
5750 {
5751 enmReflect = VMXREFLECTXCPT_DF;
5752 }
5753 else if (uIdtVector == X86_XCPT_DF)
5754 enmReflect = VMXREFLECTXCPT_TF;
5755 }
5756 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5757 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5758 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5759 {
5760 /*
5761 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5762 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5763 */
5764 enmReflect = VMXREFLECTXCPT_XCPT;
5765 }
5766 }
5767 else
5768 {
5769 /*
5770 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5771 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5772 * original exception to the guest after handling the VM-exit.
5773 */
5774 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5775 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5776 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5777 {
5778 enmReflect = VMXREFLECTXCPT_XCPT;
5779 }
5780 }
5781
5782 switch (enmReflect)
5783 {
5784 case VMXREFLECTXCPT_XCPT:
5785 {
5786 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5787 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5788 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5789
5790 uint32_t u32ErrCode = 0;
5791 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5792 {
5793 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5794 AssertRCReturn(rc, rc);
5795 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5796 }
5797
5798 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5799 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5800 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5801 rc = VINF_SUCCESS;
5802 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5803 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5804
5805 break;
5806 }
5807
5808 case VMXREFLECTXCPT_DF:
5809 {
5810 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5811 rc = VINF_HM_DOUBLE_FAULT;
5812 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5813 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5814
5815 break;
5816 }
5817
5818 case VMXREFLECTXCPT_TF:
5819 {
5820 rc = VINF_EM_RESET;
5821 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5822 uExitVector));
5823 break;
5824 }
5825
5826 default:
5827 Assert(rc == VINF_SUCCESS);
5828 break;
5829 }
5830 }
5831 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5832 return rc;
5833}
5834
5835
5836/**
5837 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5838 *
5839 * @returns VBox status code.
5840 * @param pVCpu Pointer to the VMCPU.
5841 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5842 * out-of-sync. Make sure to update the required fields
5843 * before using them.
5844 *
5845 * @remarks No-long-jump zone!!!
5846 */
5847static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5848{
5849 NOREF(pMixedCtx);
5850
5851 /*
5852 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5853 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5854 */
5855 VMMRZCallRing3Disable(pVCpu);
5856 HM_DISABLE_PREEMPT_IF_NEEDED();
5857
5858 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5859 {
5860 uint32_t uVal = 0;
5861 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5862 AssertRCReturn(rc, rc);
5863
5864 uint32_t uShadow = 0;
5865 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5866 AssertRCReturn(rc, rc);
5867
5868 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5869 CPUMSetGuestCR0(pVCpu, uVal);
5870 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5871 }
5872
5873 HM_RESTORE_PREEMPT_IF_NEEDED();
5874 VMMRZCallRing3Enable(pVCpu);
5875 return VINF_SUCCESS;
5876}
5877
5878
5879/**
5880 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5881 *
5882 * @returns VBox status code.
5883 * @param pVCpu Pointer to the VMCPU.
5884 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5885 * out-of-sync. Make sure to update the required fields
5886 * before using them.
5887 *
5888 * @remarks No-long-jump zone!!!
5889 */
5890static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5891{
5892 NOREF(pMixedCtx);
5893
5894 int rc = VINF_SUCCESS;
5895 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5896 {
5897 uint32_t uVal = 0;
5898 uint32_t uShadow = 0;
5899 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5900 AssertRCReturn(rc, rc);
5901 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5902 AssertRCReturn(rc, rc);
5903
5904 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5905 CPUMSetGuestCR4(pVCpu, uVal);
5906 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5907 }
5908 return rc;
5909}
5910
5911
5912/**
5913 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5914 *
5915 * @returns VBox status code.
5916 * @param pVCpu Pointer to the VMCPU.
5917 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5918 * out-of-sync. Make sure to update the required fields
5919 * before using them.
5920 *
5921 * @remarks No-long-jump zone!!!
5922 */
5923static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5924{
5925 int rc = VINF_SUCCESS;
5926 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5927 {
5928 uint64_t u64Val = 0;
5929 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5930 AssertRCReturn(rc, rc);
5931
5932 pMixedCtx->rip = u64Val;
5933 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5934 }
5935 return rc;
5936}
5937
5938
5939/**
5940 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5941 *
5942 * @returns VBox status code.
5943 * @param pVCpu Pointer to the VMCPU.
5944 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5945 * out-of-sync. Make sure to update the required fields
5946 * before using them.
5947 *
5948 * @remarks No-long-jump zone!!!
5949 */
5950static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5951{
5952 int rc = VINF_SUCCESS;
5953 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5954 {
5955 uint64_t u64Val = 0;
5956 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5957 AssertRCReturn(rc, rc);
5958
5959 pMixedCtx->rsp = u64Val;
5960 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5961 }
5962 return rc;
5963}
5964
5965
5966/**
5967 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5968 *
5969 * @returns VBox status code.
5970 * @param pVCpu Pointer to the VMCPU.
5971 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5972 * out-of-sync. Make sure to update the required fields
5973 * before using them.
5974 *
5975 * @remarks No-long-jump zone!!!
5976 */
5977static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5978{
5979 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5980 {
5981 uint32_t uVal = 0;
5982 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5983 AssertRCReturn(rc, rc);
5984
5985 pMixedCtx->eflags.u32 = uVal;
5986 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5987 {
5988 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5989 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5990
5991 pMixedCtx->eflags.Bits.u1VM = 0;
5992 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5993 }
5994
5995 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5996 }
5997 return VINF_SUCCESS;
5998}
5999
6000
6001/**
6002 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6003 * guest-CPU context.
6004 */
6005DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6006{
6007 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6008 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6009 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6010 return rc;
6011}
6012
6013
6014/**
6015 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6016 * from the guest-state area in the VMCS.
6017 *
6018 * @param pVCpu Pointer to the VMCPU.
6019 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6020 * out-of-sync. Make sure to update the required fields
6021 * before using them.
6022 *
6023 * @remarks No-long-jump zone!!!
6024 */
6025static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6026{
6027 uint32_t uIntrState = 0;
6028 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6029 AssertRC(rc);
6030
6031 if (!uIntrState)
6032 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6033 else
6034 {
6035 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
6036 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6037 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6038 AssertRC(rc);
6039 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6040 AssertRC(rc);
6041
6042 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6043 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6044 }
6045}
6046
6047
6048/**
6049 * Saves the guest's activity state.
6050 *
6051 * @returns VBox status code.
6052 * @param pVCpu Pointer to the VMCPU.
6053 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6054 * out-of-sync. Make sure to update the required fields
6055 * before using them.
6056 *
6057 * @remarks No-long-jump zone!!!
6058 */
6059static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6060{
6061 NOREF(pMixedCtx);
6062 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6063 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6064 return VINF_SUCCESS;
6065}
6066
6067
6068/**
6069 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6070 * the current VMCS into the guest-CPU context.
6071 *
6072 * @returns VBox status code.
6073 * @param pVCpu Pointer to the VMCPU.
6074 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6075 * out-of-sync. Make sure to update the required fields
6076 * before using them.
6077 *
6078 * @remarks No-long-jump zone!!!
6079 */
6080static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6081{
6082 int rc = VINF_SUCCESS;
6083 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6084 {
6085 uint32_t u32Val = 0;
6086 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6087 pMixedCtx->SysEnter.cs = u32Val;
6088 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6089 }
6090
6091 uint64_t u64Val = 0;
6092 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6093 {
6094 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6095 pMixedCtx->SysEnter.eip = u64Val;
6096 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6097 }
6098 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6099 {
6100 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6101 pMixedCtx->SysEnter.esp = u64Val;
6102 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6103 }
6104 return rc;
6105}
6106
6107
6108/**
6109 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6110 * the CPU back into the guest-CPU context.
6111 *
6112 * @returns VBox status code.
6113 * @param pVCpu Pointer to the VMCPU.
6114 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6115 * out-of-sync. Make sure to update the required fields
6116 * before using them.
6117 *
6118 * @remarks No-long-jump zone!!!
6119 */
6120static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6121{
6122#if HC_ARCH_BITS == 64
6123 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6124 {
6125 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6126 VMMRZCallRing3Disable(pVCpu);
6127 HM_DISABLE_PREEMPT_IF_NEEDED();
6128
6129 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6130 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6131 {
6132 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6133 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6134 }
6135
6136 HM_RESTORE_PREEMPT_IF_NEEDED();
6137 VMMRZCallRing3Enable(pVCpu);
6138 }
6139 else
6140 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6141#else
6142 NOREF(pMixedCtx);
6143 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6144#endif
6145
6146 return VINF_SUCCESS;
6147}
6148
6149
6150/**
6151 * Saves the auto load/store'd guest MSRs from the current VMCS into
6152 * the guest-CPU context.
6153 *
6154 * @returns VBox status code.
6155 * @param pVCpu Pointer to the VMCPU.
6156 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6157 * out-of-sync. Make sure to update the required fields
6158 * before using them.
6159 *
6160 * @remarks No-long-jump zone!!!
6161 */
6162static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6163{
6164 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6165 return VINF_SUCCESS;
6166
6167 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6168 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6169 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6170 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6171 {
6172 switch (pMsr->u32Msr)
6173 {
6174 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6175 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6176 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6177 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6178 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6179 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6180 break;
6181
6182 default:
6183 {
6184 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6185 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6186 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6187 }
6188 }
6189 }
6190
6191 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6192 return VINF_SUCCESS;
6193}
6194
6195
6196/**
6197 * Saves the guest control registers from the current VMCS into the guest-CPU
6198 * context.
6199 *
6200 * @returns VBox status code.
6201 * @param pVCpu Pointer to the VMCPU.
6202 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6203 * out-of-sync. Make sure to update the required fields
6204 * before using them.
6205 *
6206 * @remarks No-long-jump zone!!!
6207 */
6208static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6209{
6210 /* Guest CR0. Guest FPU. */
6211 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6212 AssertRCReturn(rc, rc);
6213
6214 /* Guest CR4. */
6215 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6216 AssertRCReturn(rc, rc);
6217
6218 /* Guest CR2 - updated always during the world-switch or in #PF. */
6219 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6220 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6221 {
6222 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6223 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6224
6225 PVM pVM = pVCpu->CTX_SUFF(pVM);
6226 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6227 || ( pVM->hm.s.fNestedPaging
6228 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6229 {
6230 uint64_t u64Val = 0;
6231 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6232 if (pMixedCtx->cr3 != u64Val)
6233 {
6234 CPUMSetGuestCR3(pVCpu, u64Val);
6235 if (VMMRZCallRing3IsEnabled(pVCpu))
6236 {
6237 PGMUpdateCR3(pVCpu, u64Val);
6238 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6239 }
6240 else
6241 {
6242 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6243 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6244 }
6245 }
6246
6247 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6248 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6249 {
6250 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6251 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6252 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6253 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6254
6255 if (VMMRZCallRing3IsEnabled(pVCpu))
6256 {
6257 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6258 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6259 }
6260 else
6261 {
6262 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6263 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6264 }
6265 }
6266 }
6267
6268 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6269 }
6270
6271 /*
6272 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6273 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6274 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6275 *
6276 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6277 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6278 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6279 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6280 *
6281 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6282 */
6283 if (VMMRZCallRing3IsEnabled(pVCpu))
6284 {
6285 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6286 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6287
6288 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6289 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6290
6291 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6292 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6293 }
6294
6295 return rc;
6296}
6297
6298
6299/**
6300 * Reads a guest segment register from the current VMCS into the guest-CPU
6301 * context.
6302 *
6303 * @returns VBox status code.
6304 * @param pVCpu Pointer to the VMCPU.
6305 * @param idxSel Index of the selector in the VMCS.
6306 * @param idxLimit Index of the segment limit in the VMCS.
6307 * @param idxBase Index of the segment base in the VMCS.
6308 * @param idxAccess Index of the access rights of the segment in the VMCS.
6309 * @param pSelReg Pointer to the segment selector.
6310 *
6311 * @remarks No-long-jump zone!!!
6312 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6313 * macro as that takes care of whether to read from the VMCS cache or
6314 * not.
6315 */
6316DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6317 PCPUMSELREG pSelReg)
6318{
6319 NOREF(pVCpu);
6320
6321 uint32_t u32Val = 0;
6322 int rc = VMXReadVmcs32(idxSel, &u32Val);
6323 AssertRCReturn(rc, rc);
6324 pSelReg->Sel = (uint16_t)u32Val;
6325 pSelReg->ValidSel = (uint16_t)u32Val;
6326 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6327
6328 rc = VMXReadVmcs32(idxLimit, &u32Val);
6329 AssertRCReturn(rc, rc);
6330 pSelReg->u32Limit = u32Val;
6331
6332 uint64_t u64Val = 0;
6333 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6334 AssertRCReturn(rc, rc);
6335 pSelReg->u64Base = u64Val;
6336
6337 rc = VMXReadVmcs32(idxAccess, &u32Val);
6338 AssertRCReturn(rc, rc);
6339 pSelReg->Attr.u = u32Val;
6340
6341 /*
6342 * If VT-x marks the segment as unusable, most other bits remain undefined:
6343 * - For CS the L, D and G bits have meaning.
6344 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6345 * - For the remaining data segments no bits are defined.
6346 *
6347 * The present bit and the unusable bit has been observed to be set at the
6348 * same time (the selector was supposed to invalid as we started executing
6349 * a V8086 interrupt in ring-0).
6350 *
6351 * What should be important for the rest of the VBox code, is that the P bit is
6352 * cleared. Some of the other VBox code recognizes the unusable bit, but
6353 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6354 * safe side here, we'll strip off P and other bits we don't care about. If
6355 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6356 *
6357 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6358 */
6359 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6360 {
6361 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6362
6363 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6364 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6365 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6366
6367 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6368#ifdef DEBUG_bird
6369 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6370 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6371 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6372#endif
6373 }
6374 return VINF_SUCCESS;
6375}
6376
6377
6378#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6379# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6380 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6381 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6382#else
6383# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6384 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6385 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6386#endif
6387
6388
6389/**
6390 * Saves the guest segment registers from the current VMCS into the guest-CPU
6391 * context.
6392 *
6393 * @returns VBox status code.
6394 * @param pVCpu Pointer to the VMCPU.
6395 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6396 * out-of-sync. Make sure to update the required fields
6397 * before using them.
6398 *
6399 * @remarks No-long-jump zone!!!
6400 */
6401static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6402{
6403 /* Guest segment registers. */
6404 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6405 {
6406 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6407 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6408 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6409 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6410 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6411 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6412 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6413
6414 /* Restore segment attributes for real-on-v86 mode hack. */
6415 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6416 {
6417 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6418 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6419 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6420 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6421 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6422 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6423 }
6424 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6425 }
6426
6427 return VINF_SUCCESS;
6428}
6429
6430
6431/**
6432 * Saves the guest descriptor table registers and task register from the current
6433 * VMCS into the guest-CPU context.
6434 *
6435 * @returns VBox status code.
6436 * @param pVCpu Pointer to the VMCPU.
6437 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6438 * out-of-sync. Make sure to update the required fields
6439 * before using them.
6440 *
6441 * @remarks No-long-jump zone!!!
6442 */
6443static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6444{
6445 int rc = VINF_SUCCESS;
6446
6447 /* Guest LDTR. */
6448 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6449 {
6450 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6451 AssertRCReturn(rc, rc);
6452 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6453 }
6454
6455 /* Guest GDTR. */
6456 uint64_t u64Val = 0;
6457 uint32_t u32Val = 0;
6458 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6459 {
6460 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6461 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6462 pMixedCtx->gdtr.pGdt = u64Val;
6463 pMixedCtx->gdtr.cbGdt = u32Val;
6464 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6465 }
6466
6467 /* Guest IDTR. */
6468 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6469 {
6470 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6471 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6472 pMixedCtx->idtr.pIdt = u64Val;
6473 pMixedCtx->idtr.cbIdt = u32Val;
6474 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6475 }
6476
6477 /* Guest TR. */
6478 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6479 {
6480 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6481 AssertRCReturn(rc, rc);
6482
6483 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6484 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6485 {
6486 rc = VMXLOCAL_READ_SEG(TR, tr);
6487 AssertRCReturn(rc, rc);
6488 }
6489 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6490 }
6491 return rc;
6492}
6493
6494#undef VMXLOCAL_READ_SEG
6495
6496
6497/**
6498 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6499 * context.
6500 *
6501 * @returns VBox status code.
6502 * @param pVCpu Pointer to the VMCPU.
6503 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6504 * out-of-sync. Make sure to update the required fields
6505 * before using them.
6506 *
6507 * @remarks No-long-jump zone!!!
6508 */
6509static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6510{
6511 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6512 {
6513 if (!pVCpu->hm.s.fUsingHyperDR7)
6514 {
6515 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6516 uint32_t u32Val;
6517 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6518 pMixedCtx->dr[7] = u32Val;
6519 }
6520
6521 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6522 }
6523 return VINF_SUCCESS;
6524}
6525
6526
6527/**
6528 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6529 *
6530 * @returns VBox status code.
6531 * @param pVCpu Pointer to the VMCPU.
6532 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6533 * out-of-sync. Make sure to update the required fields
6534 * before using them.
6535 *
6536 * @remarks No-long-jump zone!!!
6537 */
6538static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6539{
6540 NOREF(pMixedCtx);
6541
6542 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6543 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6544 return VINF_SUCCESS;
6545}
6546
6547
6548/**
6549 * Saves the entire guest state from the currently active VMCS into the
6550 * guest-CPU context. This essentially VMREADs all guest-data.
6551 *
6552 * @returns VBox status code.
6553 * @param pVCpu Pointer to the VMCPU.
6554 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6555 * out-of-sync. Make sure to update the required fields
6556 * before using them.
6557 */
6558static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6559{
6560 Assert(pVCpu);
6561 Assert(pMixedCtx);
6562
6563 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6564 return VINF_SUCCESS;
6565
6566 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6567 again on the ring-3 callback path, there is no real need to. */
6568 if (VMMRZCallRing3IsEnabled(pVCpu))
6569 VMMR0LogFlushDisable(pVCpu);
6570 else
6571 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6572 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6573
6574 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6575 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6576
6577 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6578 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6579
6580 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6581 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6582
6583 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6584 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6585
6586 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6587 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6588
6589 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6590 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6591
6592 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6593 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6594
6595 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6596 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6597
6598 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6599 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6600
6601 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6602 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6603
6604 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6605 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6606
6607 if (VMMRZCallRing3IsEnabled(pVCpu))
6608 VMMR0LogFlushEnable(pVCpu);
6609
6610 return rc;
6611}
6612
6613
6614/**
6615 * Check per-VM and per-VCPU force flag actions that require us to go back to
6616 * ring-3 for one reason or another.
6617 *
6618 * @returns VBox status code (information status code included).
6619 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6620 * ring-3.
6621 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6622 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6623 * interrupts)
6624 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6625 * all EMTs to be in ring-3.
6626 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6627 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6628 * to the EM loop.
6629 *
6630 * @param pVM Pointer to the VM.
6631 * @param pVCpu Pointer to the VMCPU.
6632 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6633 * out-of-sync. Make sure to update the required fields
6634 * before using them.
6635 */
6636static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6637{
6638 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6639
6640 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6641 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6642 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6643 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6644 {
6645 /* We need the control registers now, make sure the guest-CPU context is updated. */
6646 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6647 AssertRCReturn(rc3, rc3);
6648
6649 /* Pending HM CR3 sync. */
6650 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6651 {
6652 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6653 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6654 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6655 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6656 }
6657
6658 /* Pending HM PAE PDPEs. */
6659 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6660 {
6661 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6662 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6663 }
6664
6665 /* Pending PGM C3 sync. */
6666 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6667 {
6668 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6669 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6670 if (rc2 != VINF_SUCCESS)
6671 {
6672 AssertRC(rc2);
6673 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6674 return rc2;
6675 }
6676 }
6677
6678 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6679 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6680 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6681 {
6682 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6683 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6684 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6685 return rc2;
6686 }
6687
6688 /* Pending VM request packets, such as hardware interrupts. */
6689 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6690 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6691 {
6692 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6693 return VINF_EM_PENDING_REQUEST;
6694 }
6695
6696 /* Pending PGM pool flushes. */
6697 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6698 {
6699 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6700 return VINF_PGM_POOL_FLUSH_PENDING;
6701 }
6702
6703 /* Pending DMA requests. */
6704 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6705 {
6706 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6707 return VINF_EM_RAW_TO_R3;
6708 }
6709 }
6710
6711 return VINF_SUCCESS;
6712}
6713
6714
6715/**
6716 * Converts any TRPM trap into a pending HM event. This is typically used when
6717 * entering from ring-3 (not longjmp returns).
6718 *
6719 * @param pVCpu Pointer to the VMCPU.
6720 */
6721static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6722{
6723 Assert(TRPMHasTrap(pVCpu));
6724 Assert(!pVCpu->hm.s.Event.fPending);
6725
6726 uint8_t uVector;
6727 TRPMEVENT enmTrpmEvent;
6728 RTGCUINT uErrCode;
6729 RTGCUINTPTR GCPtrFaultAddress;
6730 uint8_t cbInstr;
6731
6732 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6733 AssertRC(rc);
6734
6735 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6736 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6737 if (enmTrpmEvent == TRPM_TRAP)
6738 {
6739 switch (uVector)
6740 {
6741 case X86_XCPT_BP:
6742 case X86_XCPT_OF:
6743 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6744 break;
6745
6746 case X86_XCPT_PF:
6747 case X86_XCPT_DF:
6748 case X86_XCPT_TS:
6749 case X86_XCPT_NP:
6750 case X86_XCPT_SS:
6751 case X86_XCPT_GP:
6752 case X86_XCPT_AC:
6753 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6754 /* no break! */
6755 default:
6756 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6757 break;
6758 }
6759 }
6760 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6761 {
6762 if (uVector == X86_XCPT_NMI)
6763 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6764 else
6765 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6766 }
6767 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6768 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6769 else
6770 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6771
6772 rc = TRPMResetTrap(pVCpu);
6773 AssertRC(rc);
6774 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6775 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6776
6777 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6778 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6779}
6780
6781
6782/**
6783 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6784 * VT-x to execute any instruction.
6785 *
6786 * @param pvCpu Pointer to the VMCPU.
6787 */
6788static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6789{
6790 Assert(pVCpu->hm.s.Event.fPending);
6791
6792 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6793 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6794 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6795 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6796
6797 /* If a trap was already pending, we did something wrong! */
6798 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6799
6800 TRPMEVENT enmTrapType;
6801 switch (uVectorType)
6802 {
6803 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6804 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6805 enmTrapType = TRPM_HARDWARE_INT;
6806 break;
6807
6808 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6809 enmTrapType = TRPM_SOFTWARE_INT;
6810 break;
6811
6812 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6813 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6814 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6815 enmTrapType = TRPM_TRAP;
6816 break;
6817
6818 default:
6819 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6820 enmTrapType = TRPM_32BIT_HACK;
6821 break;
6822 }
6823
6824 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6825
6826 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6827 AssertRC(rc);
6828
6829 if (fErrorCodeValid)
6830 TRPMSetErrorCode(pVCpu, uErrorCode);
6831
6832 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6833 && uVector == X86_XCPT_PF)
6834 {
6835 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6836 }
6837 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6838 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6839 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6840 {
6841 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6842 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6843 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6844 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6845 }
6846 pVCpu->hm.s.Event.fPending = false;
6847}
6848
6849
6850/**
6851 * Does the necessary state syncing before returning to ring-3 for any reason
6852 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6853 *
6854 * @returns VBox status code.
6855 * @param pVM Pointer to the VM.
6856 * @param pVCpu Pointer to the VMCPU.
6857 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6858 * be out-of-sync. Make sure to update the required
6859 * fields before using them.
6860 * @param fSaveGuestState Whether to save the guest state or not.
6861 *
6862 * @remarks No-long-jmp zone!!!
6863 */
6864static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6865{
6866 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6867 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6868
6869 RTCPUID idCpu = RTMpCpuId();
6870 Log4Func(("HostCpuId=%u\n", idCpu));
6871
6872 /*
6873 * !!! IMPORTANT !!!
6874 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6875 */
6876
6877 /* Save the guest state if necessary. */
6878 if ( fSaveGuestState
6879 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6880 {
6881 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6882 AssertRCReturn(rc, rc);
6883 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6884 }
6885
6886 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6887 if (CPUMIsGuestFPUStateActive(pVCpu))
6888 {
6889 /* We shouldn't reload CR0 without saving it first. */
6890 if (!fSaveGuestState)
6891 {
6892 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6893 AssertRCReturn(rc, rc);
6894 }
6895 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6896 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6897 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6898 }
6899
6900 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6901#ifdef VBOX_STRICT
6902 if (CPUMIsHyperDebugStateActive(pVCpu))
6903 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6904#endif
6905 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6906 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6907 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6908 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6909
6910#if HC_ARCH_BITS == 64
6911 /* Restore host-state bits that VT-x only restores partially. */
6912 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6913 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6914 {
6915 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6916 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6917 }
6918 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6919#endif
6920
6921#if HC_ARCH_BITS == 64
6922 /* Restore the host MSRs as we're leaving VT-x context. */
6923 if ( pVM->hm.s.fAllow64BitGuests
6924 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6925 {
6926 /* We shouldn't reload the guest MSRs without saving it first. */
6927 if (!fSaveGuestState)
6928 {
6929 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6930 AssertRCReturn(rc, rc);
6931 }
6932 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6933 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6934 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6935 }
6936#endif
6937
6938 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6939 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6940
6941 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6942 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6943 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6944 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6945 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6946 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6947 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6948 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6949
6950 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6951
6952 /** @todo This partially defeats the purpose of having preemption hooks.
6953 * The problem is, deregistering the hooks should be moved to a place that
6954 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6955 * context.
6956 */
6957 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6958 {
6959 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6960 AssertRCReturn(rc, rc);
6961
6962 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6963 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6964 }
6965 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6966 NOREF(idCpu);
6967
6968 return VINF_SUCCESS;
6969}
6970
6971
6972/**
6973 * Leaves the VT-x session.
6974 *
6975 * @returns VBox status code.
6976 * @param pVM Pointer to the VM.
6977 * @param pVCpu Pointer to the VMCPU.
6978 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6979 * out-of-sync. Make sure to update the required fields
6980 * before using them.
6981 *
6982 * @remarks No-long-jmp zone!!!
6983 */
6984DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6985{
6986 HM_DISABLE_PREEMPT_IF_NEEDED();
6987 HMVMX_ASSERT_CPU_SAFE();
6988 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6989 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6990
6991 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6992 and done this from the VMXR0ThreadCtxCallback(). */
6993 if (!pVCpu->hm.s.fLeaveDone)
6994 {
6995 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6996 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6997 pVCpu->hm.s.fLeaveDone = true;
6998 }
6999
7000 /*
7001 * !!! IMPORTANT !!!
7002 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7003 */
7004
7005 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7006 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7007 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7008 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7009 VMMR0ThreadCtxHooksDeregister(pVCpu);
7010
7011 /* Leave HM context. This takes care of local init (term). */
7012 int rc = HMR0LeaveCpu(pVCpu);
7013
7014 HM_RESTORE_PREEMPT_IF_NEEDED();
7015
7016 return rc;
7017}
7018
7019
7020/**
7021 * Does the necessary state syncing before doing a longjmp to ring-3.
7022 *
7023 * @returns VBox status code.
7024 * @param pVM Pointer to the VM.
7025 * @param pVCpu Pointer to the VMCPU.
7026 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7027 * out-of-sync. Make sure to update the required fields
7028 * before using them.
7029 *
7030 * @remarks No-long-jmp zone!!!
7031 */
7032DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7033{
7034 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7035}
7036
7037
7038/**
7039 * Take necessary actions before going back to ring-3.
7040 *
7041 * An action requires us to go back to ring-3. This function does the necessary
7042 * steps before we can safely return to ring-3. This is not the same as longjmps
7043 * to ring-3, this is voluntary and prepares the guest so it may continue
7044 * executing outside HM (recompiler/IEM).
7045 *
7046 * @returns VBox status code.
7047 * @param pVM Pointer to the VM.
7048 * @param pVCpu Pointer to the VMCPU.
7049 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7050 * out-of-sync. Make sure to update the required fields
7051 * before using them.
7052 * @param rcExit The reason for exiting to ring-3. Can be
7053 * VINF_VMM_UNKNOWN_RING3_CALL.
7054 */
7055static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7056{
7057 Assert(pVM);
7058 Assert(pVCpu);
7059 Assert(pMixedCtx);
7060 HMVMX_ASSERT_PREEMPT_SAFE();
7061
7062 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7063 {
7064 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7065 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7066 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7067 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7068 }
7069
7070 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7071 VMMRZCallRing3Disable(pVCpu);
7072 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7073
7074 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7075 if (pVCpu->hm.s.Event.fPending)
7076 {
7077 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7078 Assert(!pVCpu->hm.s.Event.fPending);
7079 }
7080
7081 /* Save guest state and restore host state bits. */
7082 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7083 AssertRCReturn(rc, rc);
7084 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7085
7086 /* Sync recompiler state. */
7087 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7088 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7089 | CPUM_CHANGED_LDTR
7090 | CPUM_CHANGED_GDTR
7091 | CPUM_CHANGED_IDTR
7092 | CPUM_CHANGED_TR
7093 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7094 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7095 if ( pVM->hm.s.fNestedPaging
7096 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7097 {
7098 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7099 }
7100
7101 Assert(!pVCpu->hm.s.fClearTrapFlag);
7102
7103 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7104 if (rcExit != VINF_EM_RAW_INTERRUPT)
7105 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7106
7107 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7108
7109 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7110 VMMRZCallRing3RemoveNotification(pVCpu);
7111 VMMRZCallRing3Enable(pVCpu);
7112
7113 return rc;
7114}
7115
7116
7117/**
7118 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7119 * longjump to ring-3 and possibly get preempted.
7120 *
7121 * @returns VBox status code.
7122 * @param pVCpu Pointer to the VMCPU.
7123 * @param enmOperation The operation causing the ring-3 longjump.
7124 * @param pvUser Opaque pointer to the guest-CPU context. The data
7125 * may be out-of-sync. Make sure to update the required
7126 * fields before using them.
7127 */
7128DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7129{
7130 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7131 {
7132 /*
7133 * !!! IMPORTANT !!!
7134 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7135 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7136 */
7137 VMMRZCallRing3RemoveNotification(pVCpu);
7138 VMMRZCallRing3Disable(pVCpu);
7139 HM_DISABLE_PREEMPT_IF_NEEDED();
7140
7141 PVM pVM = pVCpu->CTX_SUFF(pVM);
7142 if (CPUMIsGuestFPUStateActive(pVCpu))
7143 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7144
7145 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7146
7147#if HC_ARCH_BITS == 64
7148 /* Restore host-state bits that VT-x only restores partially. */
7149 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7150 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7151 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7152 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7153
7154 /* Restore the host MSRs as we're leaving VT-x context. */
7155 if ( pVM->hm.s.fAllow64BitGuests
7156 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7157 {
7158 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7159 }
7160#endif
7161 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7162 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7163 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7164 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7165 {
7166 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7167 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7168 }
7169
7170 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7171 VMMR0ThreadCtxHooksDeregister(pVCpu);
7172
7173 HMR0LeaveCpu(pVCpu);
7174 HM_RESTORE_PREEMPT_IF_NEEDED();
7175 return VINF_SUCCESS;
7176 }
7177
7178 Assert(pVCpu);
7179 Assert(pvUser);
7180 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7181 HMVMX_ASSERT_PREEMPT_SAFE();
7182
7183 VMMRZCallRing3Disable(pVCpu);
7184 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7185
7186 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7187 enmOperation));
7188
7189 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7190 AssertRCReturn(rc, rc);
7191
7192 VMMRZCallRing3Enable(pVCpu);
7193 return VINF_SUCCESS;
7194}
7195
7196
7197/**
7198 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7199 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7200 *
7201 * @param pVCpu Pointer to the VMCPU.
7202 */
7203DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7204{
7205 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7206 {
7207 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7208 {
7209 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7210 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7211 AssertRC(rc);
7212 Log4(("Setup interrupt-window exiting\n"));
7213 }
7214 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7215}
7216
7217
7218/**
7219 * Clears the interrupt-window exiting control in the VMCS.
7220 *
7221 * @param pVCpu Pointer to the VMCPU.
7222 */
7223DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7224{
7225 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7226 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7227 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7228 AssertRC(rc);
7229 Log4(("Cleared interrupt-window exiting\n"));
7230}
7231
7232
7233/**
7234 * Evaluates the event to be delivered to the guest and sets it as the pending
7235 * event.
7236 *
7237 * @param pVCpu Pointer to the VMCPU.
7238 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7239 * out-of-sync. Make sure to update the required fields
7240 * before using them.
7241 */
7242static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7243{
7244 Assert(!pVCpu->hm.s.Event.fPending);
7245
7246 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7247 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7248 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7249 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7250
7251 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7252 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7253 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7254 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7255 Assert(!TRPMHasTrap(pVCpu));
7256
7257 /** @todo SMI. SMIs take priority over NMIs. */
7258 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
7259 {
7260 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7261 if ( !fBlockMovSS
7262 && !fBlockSti)
7263 {
7264 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7265 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7266 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7267 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7268
7269 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
7270 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7271 }
7272 else
7273 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7274 }
7275 /*
7276 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7277 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7278 */
7279 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7280 && !pVCpu->hm.s.fSingleInstruction)
7281 {
7282 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7283 AssertRC(rc);
7284 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7285 if ( !fBlockInt
7286 && !fBlockSti
7287 && !fBlockMovSS)
7288 {
7289 uint8_t u8Interrupt;
7290 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7291 if (RT_SUCCESS(rc))
7292 {
7293 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7294 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7295 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7296
7297 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7298 }
7299 else
7300 {
7301 /** @todo Does this actually happen? If not turn it into an assertion. */
7302 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7303 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7304 }
7305 }
7306 else
7307 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7308 }
7309}
7310
7311
7312/**
7313 * Sets a pending-debug exception to be delivered to the guest if the guest is
7314 * single-stepping.
7315 *
7316 * @param pVCpu Pointer to the VMCPU.
7317 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7318 * out-of-sync. Make sure to update the required fields
7319 * before using them.
7320 */
7321DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7322{
7323 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7324 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7325 {
7326 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7327 AssertRC(rc);
7328 }
7329}
7330
7331
7332/**
7333 * Injects any pending events into the guest if the guest is in a state to
7334 * receive them.
7335 *
7336 * @returns VBox status code (informational status codes included).
7337 * @param pVCpu Pointer to the VMCPU.
7338 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7339 * out-of-sync. Make sure to update the required fields
7340 * before using them.
7341 */
7342static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7343{
7344 HMVMX_ASSERT_PREEMPT_SAFE();
7345 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7346
7347 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7348 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7349 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7350 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7351
7352 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7353 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7354 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7355 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7356 Assert(!TRPMHasTrap(pVCpu));
7357
7358 int rc = VINF_SUCCESS;
7359 if (pVCpu->hm.s.Event.fPending)
7360 {
7361 /*
7362 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7363 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7364 * ended up enabling interrupts outside VT-x.
7365 */
7366 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7367 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7368 && ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT
7369 || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI))
7370 {
7371 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7372 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7373 }
7374#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
7375 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7376 {
7377 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7378 AssertRCReturn(rc, rc);
7379 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7380 Assert(!fBlockInt);
7381 Assert(!fBlockSti);
7382 Assert(!fBlockMovSS);
7383 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7384 }
7385 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7386 {
7387 Assert(!fBlockSti);
7388 Assert(!fBlockMovSS);
7389 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7390 }
7391#endif
7392 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7393 (uint8_t)uIntType));
7394 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7395 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7396 AssertRCReturn(rc, rc);
7397
7398 /* Update the interruptibility-state as it could have been changed by
7399 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7400 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7401 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7402
7403#ifdef VBOX_WITH_STATISTICS
7404 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7405 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7406 else
7407 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7408#endif
7409 }
7410
7411 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7412 if ( fBlockSti
7413 || fBlockMovSS)
7414 {
7415 if ( !pVCpu->hm.s.fSingleInstruction
7416 && !DBGFIsStepping(pVCpu))
7417 {
7418 /*
7419 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7420 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7421 * See Intel spec. 27.3.4 "Saving Non-Register State".
7422 */
7423 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7424 AssertRCReturn(rc2, rc2);
7425 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7426 }
7427 else if (pMixedCtx->eflags.Bits.u1TF)
7428 {
7429 /*
7430 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7431 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7432 */
7433 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7434 uIntrState = 0;
7435 }
7436 }
7437
7438 /*
7439 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7440 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7441 */
7442 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7443 AssertRC(rc2);
7444
7445 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7446 NOREF(fBlockMovSS); NOREF(fBlockSti);
7447 return rc;
7448}
7449
7450
7451/**
7452 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7453 *
7454 * @param pVCpu Pointer to the VMCPU.
7455 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7456 * out-of-sync. Make sure to update the required fields
7457 * before using them.
7458 */
7459DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7460{
7461 NOREF(pMixedCtx);
7462 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7463 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7464}
7465
7466
7467/**
7468 * Injects a double-fault (#DF) exception into the VM.
7469 *
7470 * @returns VBox status code (informational status code included).
7471 * @param pVCpu Pointer to the VMCPU.
7472 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7473 * out-of-sync. Make sure to update the required fields
7474 * before using them.
7475 */
7476DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7477{
7478 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7479 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7480 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7481 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7482 puIntrState);
7483}
7484
7485
7486/**
7487 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7488 *
7489 * @param pVCpu Pointer to the VMCPU.
7490 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7491 * out-of-sync. Make sure to update the required fields
7492 * before using them.
7493 */
7494DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7495{
7496 NOREF(pMixedCtx);
7497 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7498 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7499 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7500}
7501
7502
7503/**
7504 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7505 *
7506 * @param pVCpu Pointer to the VMCPU.
7507 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7508 * out-of-sync. Make sure to update the required fields
7509 * before using them.
7510 * @param cbInstr The value of RIP that is to be pushed on the guest
7511 * stack.
7512 */
7513DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7514{
7515 NOREF(pMixedCtx);
7516 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7517 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7518 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7519}
7520
7521
7522/**
7523 * Injects a general-protection (#GP) fault into the VM.
7524 *
7525 * @returns VBox status code (informational status code included).
7526 * @param pVCpu Pointer to the VMCPU.
7527 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7528 * out-of-sync. Make sure to update the required fields
7529 * before using them.
7530 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7531 * mode, i.e. in real-mode it's not valid).
7532 * @param u32ErrorCode The error code associated with the #GP.
7533 */
7534DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7535 uint32_t *puIntrState)
7536{
7537 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7538 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7539 if (fErrorCodeValid)
7540 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7541 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7542 puIntrState);
7543}
7544
7545
7546/**
7547 * Sets a general-protection (#GP) exception as pending-for-injection into the
7548 * VM.
7549 *
7550 * @param pVCpu Pointer to the VMCPU.
7551 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7552 * out-of-sync. Make sure to update the required fields
7553 * before using them.
7554 * @param u32ErrorCode The error code associated with the #GP.
7555 */
7556DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7557{
7558 NOREF(pMixedCtx);
7559 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7560 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7561 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7562 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7563}
7564
7565
7566/**
7567 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7568 *
7569 * @param pVCpu Pointer to the VMCPU.
7570 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7571 * out-of-sync. Make sure to update the required fields
7572 * before using them.
7573 * @param uVector The software interrupt vector number.
7574 * @param cbInstr The value of RIP that is to be pushed on the guest
7575 * stack.
7576 */
7577DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7578{
7579 NOREF(pMixedCtx);
7580 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7581 if ( uVector == X86_XCPT_BP
7582 || uVector == X86_XCPT_OF)
7583 {
7584 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7585 }
7586 else
7587 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7588 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7589}
7590
7591
7592/**
7593 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7594 * stack.
7595 *
7596 * @returns VBox status code (information status code included).
7597 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7598 * @param pVM Pointer to the VM.
7599 * @param pMixedCtx Pointer to the guest-CPU context.
7600 * @param uValue The value to push to the guest stack.
7601 */
7602DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7603{
7604 /*
7605 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7606 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7607 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7608 */
7609 if (pMixedCtx->sp == 1)
7610 return VINF_EM_RESET;
7611 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7612 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7613 AssertRCReturn(rc, rc);
7614 return rc;
7615}
7616
7617
7618/**
7619 * Injects an event into the guest upon VM-entry by updating the relevant fields
7620 * in the VM-entry area in the VMCS.
7621 *
7622 * @returns VBox status code (informational error codes included).
7623 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7624 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7625 *
7626 * @param pVCpu Pointer to the VMCPU.
7627 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7628 * be out-of-sync. Make sure to update the required
7629 * fields before using them.
7630 * @param u64IntInfo The VM-entry interruption-information field.
7631 * @param cbInstr The VM-entry instruction length in bytes (for
7632 * software interrupts, exceptions and privileged
7633 * software exceptions).
7634 * @param u32ErrCode The VM-entry exception error code.
7635 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7636 * @param puIntrState Pointer to the current guest interruptibility-state.
7637 * This interruptibility-state will be updated if
7638 * necessary. This cannot not be NULL.
7639 *
7640 * @remarks Requires CR0!
7641 * @remarks No-long-jump zone!!!
7642 */
7643static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7644 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7645{
7646 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7647 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7648 Assert(puIntrState);
7649 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7650
7651 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7652 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7653
7654#ifdef VBOX_STRICT
7655 /* Validate the error-code-valid bit for hardware exceptions. */
7656 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7657 {
7658 switch (uVector)
7659 {
7660 case X86_XCPT_PF:
7661 case X86_XCPT_DF:
7662 case X86_XCPT_TS:
7663 case X86_XCPT_NP:
7664 case X86_XCPT_SS:
7665 case X86_XCPT_GP:
7666 case X86_XCPT_AC:
7667 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7668 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7669 /* fallthru */
7670 default:
7671 break;
7672 }
7673 }
7674#endif
7675
7676 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7677 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7678 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7679
7680 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7681
7682 /* We require CR0 to check if the guest is in real-mode. */
7683 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7684 AssertRCReturn(rc, rc);
7685
7686 /*
7687 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7688 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7689 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7690 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7691 */
7692 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7693 {
7694 PVM pVM = pVCpu->CTX_SUFF(pVM);
7695 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7696 {
7697 Assert(PDMVmmDevHeapIsEnabled(pVM));
7698 Assert(pVM->hm.s.vmx.pRealModeTSS);
7699
7700 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7701 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7702 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7703 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7704 AssertRCReturn(rc, rc);
7705 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7706
7707 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7708 const size_t cbIdtEntry = sizeof(X86IDTR16);
7709 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7710 {
7711 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7712 if (uVector == X86_XCPT_DF)
7713 return VINF_EM_RESET;
7714 else if (uVector == X86_XCPT_GP)
7715 {
7716 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7717 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7718 }
7719
7720 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7721 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7722 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7723 }
7724
7725 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7726 uint16_t uGuestIp = pMixedCtx->ip;
7727 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7728 {
7729 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7730 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7731 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7732 }
7733 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7734 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7735
7736 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7737 X86IDTR16 IdtEntry;
7738 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7739 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7740 AssertRCReturn(rc, rc);
7741
7742 /* Construct the stack frame for the interrupt/exception handler. */
7743 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7744 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7745 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7746 AssertRCReturn(rc, rc);
7747
7748 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7749 if (rc == VINF_SUCCESS)
7750 {
7751 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7752 pMixedCtx->rip = IdtEntry.offSel;
7753 pMixedCtx->cs.Sel = IdtEntry.uSel;
7754 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7755 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7756 && uVector == X86_XCPT_PF)
7757 {
7758 pMixedCtx->cr2 = GCPtrFaultAddress;
7759 }
7760
7761 /* If any other guest-state bits are changed here, make sure to update
7762 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7763 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7764 | HM_CHANGED_GUEST_RIP
7765 | HM_CHANGED_GUEST_RFLAGS
7766 | HM_CHANGED_GUEST_RSP);
7767
7768 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7769 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7770 {
7771 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7772 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7773 Log4(("Clearing inhibition due to STI.\n"));
7774 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7775 }
7776 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7777
7778 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7779 it, if we are returning to ring-3 before executing guest code. */
7780 pVCpu->hm.s.Event.fPending = false;
7781 }
7782 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7783 return rc;
7784 }
7785 else
7786 {
7787 /*
7788 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7789 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7790 */
7791 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7792 }
7793 }
7794
7795 /* Validate. */
7796 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7797 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7798 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7799
7800 /* Inject. */
7801 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7802 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7803 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7804 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7805
7806 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7807 && uVector == X86_XCPT_PF)
7808 {
7809 pMixedCtx->cr2 = GCPtrFaultAddress;
7810 }
7811
7812 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7813 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7814
7815 AssertRCReturn(rc, rc);
7816 return rc;
7817}
7818
7819
7820/**
7821 * Clears the interrupt-window exiting control in the VMCS and if necessary
7822 * clears the current event in the VMCS as well.
7823 *
7824 * @returns VBox status code.
7825 * @param pVCpu Pointer to the VMCPU.
7826 *
7827 * @remarks Use this function only to clear events that have not yet been
7828 * delivered to the guest but are injected in the VMCS!
7829 * @remarks No-long-jump zone!!!
7830 */
7831static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7832{
7833 int rc;
7834 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7835
7836 /* Clear interrupt-window exiting control. */
7837 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7838 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7839
7840 if (!pVCpu->hm.s.Event.fPending)
7841 return;
7842
7843#ifdef VBOX_STRICT
7844 uint32_t u32EntryInfo;
7845 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7846 AssertRC(rc);
7847 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7848#endif
7849
7850 /* Clear the entry-interruption field (including the valid bit). */
7851 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7852 AssertRC(rc);
7853
7854 /* Clear the pending debug exception field. */
7855 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7856 AssertRC(rc);
7857
7858 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
7859 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
7860}
7861
7862
7863/**
7864 * Enters the VT-x session.
7865 *
7866 * @returns VBox status code.
7867 * @param pVM Pointer to the VM.
7868 * @param pVCpu Pointer to the VMCPU.
7869 * @param pCpu Pointer to the CPU info struct.
7870 */
7871VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7872{
7873 AssertPtr(pVM);
7874 AssertPtr(pVCpu);
7875 Assert(pVM->hm.s.vmx.fSupported);
7876 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7877 NOREF(pCpu); NOREF(pVM);
7878
7879 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7880 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7881
7882#ifdef VBOX_STRICT
7883 /* Make sure we're in VMX root mode. */
7884 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7885 if (!(u32HostCR4 & X86_CR4_VMXE))
7886 {
7887 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7888 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7889 }
7890#endif
7891
7892 /*
7893 * Load the VCPU's VMCS as the current (and active) one.
7894 */
7895 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7896 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7897 if (RT_FAILURE(rc))
7898 return rc;
7899
7900 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7901 pVCpu->hm.s.fLeaveDone = false;
7902 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7903
7904 return VINF_SUCCESS;
7905}
7906
7907
7908/**
7909 * The thread-context callback (only on platforms which support it).
7910 *
7911 * @param enmEvent The thread-context event.
7912 * @param pVCpu Pointer to the VMCPU.
7913 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7914 * @thread EMT(pVCpu)
7915 */
7916VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7917{
7918 NOREF(fGlobalInit);
7919
7920 switch (enmEvent)
7921 {
7922 case RTTHREADCTXEVENT_PREEMPTING:
7923 {
7924 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7925 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7926 VMCPU_ASSERT_EMT(pVCpu);
7927
7928 PVM pVM = pVCpu->CTX_SUFF(pVM);
7929 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7930
7931 /* No longjmps (logger flushes, locks) in this fragile context. */
7932 VMMRZCallRing3Disable(pVCpu);
7933 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7934
7935 /*
7936 * Restore host-state (FPU, debug etc.)
7937 */
7938 if (!pVCpu->hm.s.fLeaveDone)
7939 {
7940 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7941 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7942 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7943 pVCpu->hm.s.fLeaveDone = true;
7944 }
7945
7946 /* Leave HM context, takes care of local init (term). */
7947 int rc = HMR0LeaveCpu(pVCpu);
7948 AssertRC(rc); NOREF(rc);
7949
7950 /* Restore longjmp state. */
7951 VMMRZCallRing3Enable(pVCpu);
7952 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7953 break;
7954 }
7955
7956 case RTTHREADCTXEVENT_RESUMED:
7957 {
7958 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7959 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7960 VMCPU_ASSERT_EMT(pVCpu);
7961
7962 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7963 VMMRZCallRing3Disable(pVCpu);
7964 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7965
7966 /* Initialize the bare minimum state required for HM. This takes care of
7967 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7968 int rc = HMR0EnterCpu(pVCpu);
7969 AssertRC(rc);
7970 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7971
7972 /* Load the active VMCS as the current one. */
7973 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7974 {
7975 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7976 AssertRC(rc); NOREF(rc);
7977 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7978 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7979 }
7980 pVCpu->hm.s.fLeaveDone = false;
7981
7982 /* Restore longjmp state. */
7983 VMMRZCallRing3Enable(pVCpu);
7984 break;
7985 }
7986
7987 default:
7988 break;
7989 }
7990}
7991
7992
7993/**
7994 * Saves the host state in the VMCS host-state.
7995 * Sets up the VM-exit MSR-load area.
7996 *
7997 * The CPU state will be loaded from these fields on every successful VM-exit.
7998 *
7999 * @returns VBox status code.
8000 * @param pVM Pointer to the VM.
8001 * @param pVCpu Pointer to the VMCPU.
8002 *
8003 * @remarks No-long-jump zone!!!
8004 */
8005static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8006{
8007 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8008
8009 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8010 return VINF_SUCCESS;
8011
8012 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8013 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8014
8015 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8016 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8017
8018 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8019 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8020
8021 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8022 return rc;
8023}
8024
8025
8026/**
8027 * Saves the host state in the VMCS host-state.
8028 *
8029 * @returns VBox status code.
8030 * @param pVM Pointer to the VM.
8031 * @param pVCpu Pointer to the VMCPU.
8032 *
8033 * @remarks No-long-jump zone!!!
8034 */
8035VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8036{
8037 AssertPtr(pVM);
8038 AssertPtr(pVCpu);
8039
8040 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8041
8042 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8043 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8044 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8045 return hmR0VmxSaveHostState(pVM, pVCpu);
8046}
8047
8048
8049/**
8050 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8051 * loaded from these fields on every successful VM-entry.
8052 *
8053 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8054 * Sets up the VM-entry controls.
8055 * Sets up the appropriate VMX non-root function to execute guest code based on
8056 * the guest CPU mode.
8057 *
8058 * @returns VBox status code.
8059 * @param pVM Pointer to the VM.
8060 * @param pVCpu Pointer to the VMCPU.
8061 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8062 * out-of-sync. Make sure to update the required fields
8063 * before using them.
8064 *
8065 * @remarks No-long-jump zone!!!
8066 */
8067static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8068{
8069 AssertPtr(pVM);
8070 AssertPtr(pVCpu);
8071 AssertPtr(pMixedCtx);
8072 HMVMX_ASSERT_PREEMPT_SAFE();
8073
8074#ifdef LOG_ENABLED
8075 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8076 * probably not initialized yet? Anyway this will do for now.
8077 *
8078 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8079 * interface and disable ring-3 calls when thread-context hooks are not
8080 * available. */
8081 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8082 VMMR0LogFlushDisable(pVCpu);
8083#endif
8084
8085 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8086
8087 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8088
8089 /* Determine real-on-v86 mode. */
8090 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8091 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8092 && CPUMIsGuestInRealModeEx(pMixedCtx))
8093 {
8094 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8095 }
8096
8097 /*
8098 * Load the guest-state into the VMCS.
8099 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8100 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8101 */
8102 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8103 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8104
8105 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8106 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8107 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8108
8109 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8110 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8111 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8112
8113 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8114 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8115
8116 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8117 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8118
8119 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8120 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8121 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8122
8123 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8124 determine we don't have to swap EFER after all. */
8125 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8126 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8127
8128 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8129 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8130
8131 /*
8132 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8133 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8134 */
8135 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8136 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8137
8138 /* Clear any unused and reserved bits. */
8139 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8140
8141#ifdef LOG_ENABLED
8142 /* Only reenable log-flushing if the caller has it enabled. */
8143 if (!fCallerDisabledLogFlush)
8144 VMMR0LogFlushEnable(pVCpu);
8145#endif
8146
8147 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8148 return rc;
8149}
8150
8151
8152/**
8153 * Loads the state shared between the host and guest into the VMCS.
8154 *
8155 * @param pVM Pointer to the VM.
8156 * @param pVCpu Pointer to the VMCPU.
8157 * @param pCtx Pointer to the guest-CPU context.
8158 *
8159 * @remarks No-long-jump zone!!!
8160 */
8161static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8162{
8163 NOREF(pVM);
8164
8165 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8166 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8167
8168 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8169 {
8170 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8171 AssertRC(rc);
8172 }
8173
8174 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8175 {
8176 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8177 AssertRC(rc);
8178
8179 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8180 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8181 {
8182 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8183 AssertRC(rc);
8184 }
8185 }
8186
8187 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8188 {
8189#if HC_ARCH_BITS == 64
8190 if (pVM->hm.s.fAllow64BitGuests)
8191 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8192#endif
8193 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8194 }
8195
8196 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8197 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8198}
8199
8200
8201/**
8202 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8203 *
8204 * @param pVM Pointer to the VM.
8205 * @param pVCpu Pointer to the VMCPU.
8206 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8207 * out-of-sync. Make sure to update the required fields
8208 * before using them.
8209 */
8210DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8211{
8212 HMVMX_ASSERT_PREEMPT_SAFE();
8213
8214 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8215#ifdef HMVMX_SYNC_FULL_GUEST_STATE
8216 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8217#endif
8218
8219 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8220 {
8221 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8222 AssertRC(rc);
8223 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8224 }
8225 else if (HMCPU_CF_VALUE(pVCpu))
8226 {
8227 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8228 AssertRC(rc);
8229 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8230 }
8231
8232 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8233 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8234 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8235 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8236}
8237
8238
8239/**
8240 * Does the preparations before executing guest code in VT-x.
8241 *
8242 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8243 * recompiler. We must be cautious what we do here regarding committing
8244 * guest-state information into the VMCS assuming we assuredly execute the
8245 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8246 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8247 * so that the recompiler can (and should) use them when it resumes guest
8248 * execution. Otherwise such operations must be done when we can no longer
8249 * exit to ring-3.
8250 *
8251 * @returns Strict VBox status code.
8252 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8253 * have been disabled.
8254 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8255 * double-fault into the guest.
8256 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8257 *
8258 * @param pVM Pointer to the VM.
8259 * @param pVCpu Pointer to the VMCPU.
8260 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8261 * out-of-sync. Make sure to update the required fields
8262 * before using them.
8263 * @param pVmxTransient Pointer to the VMX transient structure.
8264 */
8265static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8266{
8267 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8268
8269#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8270 PGMRZDynMapFlushAutoSet(pVCpu);
8271#endif
8272
8273 /* Check force flag actions that might require us to go back to ring-3. */
8274 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8275 if (rc != VINF_SUCCESS)
8276 return rc;
8277
8278#ifndef IEM_VERIFICATION_MODE_FULL
8279 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8280 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8281 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8282 {
8283 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8284 RTGCPHYS GCPhysApicBase;
8285 GCPhysApicBase = pMixedCtx->msrApicBase;
8286 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8287
8288 /* Unalias any existing mapping. */
8289 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8290 AssertRCReturn(rc, rc);
8291
8292 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8293 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8294 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8295 AssertRCReturn(rc, rc);
8296
8297 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8298 }
8299#endif /* !IEM_VERIFICATION_MODE_FULL */
8300
8301 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
8302 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8303
8304 /*
8305 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8306 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8307 */
8308 if (TRPMHasTrap(pVCpu))
8309 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8310 else if (!pVCpu->hm.s.Event.fPending)
8311 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8312
8313 /*
8314 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8315 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8316 */
8317 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8318 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8319 {
8320 Assert(rc == VINF_EM_RESET);
8321 return rc;
8322 }
8323
8324 /*
8325 * No longjmps to ring-3 from this point on!!!
8326 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8327 * This also disables flushing of the R0-logger instance (if any).
8328 */
8329 VMMRZCallRing3Disable(pVCpu);
8330
8331 /*
8332 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8333 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8334 *
8335 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8336 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8337 *
8338 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8339 * executing guest code.
8340 */
8341 pVmxTransient->uEflags = ASMIntDisableFlags();
8342 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8343 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8344 {
8345 hmR0VmxClearEventVmcs(pVCpu);
8346 ASMSetFlags(pVmxTransient->uEflags);
8347 VMMRZCallRing3Enable(pVCpu);
8348 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8349 return VINF_EM_RAW_TO_R3;
8350 }
8351
8352 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8353 {
8354 hmR0VmxClearEventVmcs(pVCpu);
8355 ASMSetFlags(pVmxTransient->uEflags);
8356 VMMRZCallRing3Enable(pVCpu);
8357 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8358 return VINF_EM_RAW_INTERRUPT;
8359 }
8360
8361 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8362 pVCpu->hm.s.Event.fPending = false;
8363
8364 return VINF_SUCCESS;
8365}
8366
8367
8368/**
8369 * Prepares to run guest code in VT-x and we've committed to doing so. This
8370 * means there is no backing out to ring-3 or anywhere else at this
8371 * point.
8372 *
8373 * @param pVM Pointer to the VM.
8374 * @param pVCpu Pointer to the VMCPU.
8375 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8376 * out-of-sync. Make sure to update the required fields
8377 * before using them.
8378 * @param pVmxTransient Pointer to the VMX transient structure.
8379 *
8380 * @remarks Called with preemption disabled.
8381 * @remarks No-long-jump zone!!!
8382 */
8383static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8384{
8385 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8386 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8387 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8388
8389 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8390 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8391
8392 /*
8393 * If we are injecting events to a real-on-v86 mode guest, we may have to update
8394 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8395 * Reload only the necessary state, the assertion will catch if other parts of the code
8396 * change.
8397 */
8398 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8399 {
8400 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8401 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8402 }
8403
8404#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8405 if (!CPUMIsGuestFPUStateActive(pVCpu))
8406 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8407 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8408#endif
8409
8410 if ( pVCpu->hm.s.fUseGuestFpu
8411 && !CPUMIsGuestFPUStateActive(pVCpu))
8412 {
8413 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8414 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8415 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8416 }
8417
8418 /*
8419 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8420 */
8421 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8422 && pVCpu->hm.s.vmx.cMsrs > 0)
8423 {
8424 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8425 }
8426
8427 /*
8428 * Load the host state bits as we may've been preempted (only happens when
8429 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8430 */
8431 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8432 {
8433 /* This ASSUMES that pfnStartVM has been set up already. */
8434 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8435 AssertRC(rc);
8436 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8437 }
8438 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8439
8440 /*
8441 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8442 */
8443 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8444 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8445 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8446
8447 /* Store status of the shared guest-host state at the time of VM-entry. */
8448#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8449 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8450 {
8451 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8452 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8453 }
8454 else
8455#endif
8456 {
8457 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8458 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8459 }
8460 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8461
8462 /*
8463 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8464 */
8465 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8466 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8467
8468 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8469 RTCPUID idCurrentCpu = pCpu->idCpu;
8470 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8471 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8472 {
8473 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8474 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8475 }
8476
8477 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8478 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8479 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8480 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8481
8482 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8483
8484 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8485 to start executing. */
8486
8487 /*
8488 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8489 */
8490 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8491 {
8492 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8493 {
8494 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8495 AssertRC(rc2);
8496 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8497 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8498 true /* fUpdateHostMsr */);
8499 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8500 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8501 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8502 }
8503 else
8504 {
8505 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8506 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8507 }
8508 }
8509
8510#ifdef VBOX_STRICT
8511 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8512 hmR0VmxCheckHostEferMsr(pVCpu);
8513 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8514#endif
8515#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8516 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8517 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8518 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8519#endif
8520}
8521
8522
8523/**
8524 * Performs some essential restoration of state after running guest code in
8525 * VT-x.
8526 *
8527 * @param pVM Pointer to the VM.
8528 * @param pVCpu Pointer to the VMCPU.
8529 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8530 * out-of-sync. Make sure to update the required fields
8531 * before using them.
8532 * @param pVmxTransient Pointer to the VMX transient structure.
8533 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8534 *
8535 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8536 *
8537 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8538 * unconditionally when it is safe to do so.
8539 */
8540static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8541{
8542 NOREF(pVM);
8543
8544 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8545
8546 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8547 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8548 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8549 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8550 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8551
8552 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8553 {
8554 /** @todo Find a way to fix hardcoding a guestimate. */
8555 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8556 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8557 }
8558
8559 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8560 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8561 Assert(!(ASMGetFlags() & X86_EFL_IF));
8562 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8563
8564#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8565 if (CPUMIsGuestFPUStateActive(pVCpu))
8566 {
8567 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8568 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8569 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8570 }
8571#endif
8572
8573#if HC_ARCH_BITS == 64
8574 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8575#endif
8576 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8577#ifdef VBOX_STRICT
8578 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8579#endif
8580 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8581 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8582
8583 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8584 uint32_t uExitReason;
8585 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8586 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8587 AssertRC(rc);
8588 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8589 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8590
8591 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8592 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8593 {
8594 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8595 pVmxTransient->fVMEntryFailed));
8596 return;
8597 }
8598
8599 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8600 {
8601 /* Update the guest interruptibility-state from the VMCS. */
8602 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8603#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
8604 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8605 AssertRC(rc);
8606#endif
8607 /*
8608 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8609 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8610 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8611 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8612 */
8613 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8614 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8615 {
8616 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8617 AssertRC(rc);
8618 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8619 }
8620 }
8621}
8622
8623
8624/**
8625 * Runs the guest code using VT-x the normal way.
8626 *
8627 * @returns VBox status code.
8628 * @param pVM Pointer to the VM.
8629 * @param pVCpu Pointer to the VMCPU.
8630 * @param pCtx Pointer to the guest-CPU context.
8631 *
8632 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8633 */
8634static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8635{
8636 VMXTRANSIENT VmxTransient;
8637 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8638 int rc = VERR_INTERNAL_ERROR_5;
8639 uint32_t cLoops = 0;
8640
8641 for (;; cLoops++)
8642 {
8643 Assert(!HMR0SuspendPending());
8644 HMVMX_ASSERT_CPU_SAFE();
8645
8646 /* Preparatory work for running guest code, this may force us to return
8647 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8648 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8649 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8650 if (rc != VINF_SUCCESS)
8651 break;
8652
8653 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8654 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8655 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8656
8657 /* Restore any residual host-state and save any bits shared between host
8658 and guest into the guest-CPU state. Re-enables interrupts! */
8659 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8660
8661 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8662 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8663 {
8664 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8665 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8666 return rc;
8667 }
8668
8669 /* Handle the VM-exit. */
8670 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8671 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8672 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8673 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8674 HMVMX_START_EXIT_DISPATCH_PROF();
8675#ifdef HMVMX_USE_FUNCTION_TABLE
8676 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8677#else
8678 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8679#endif
8680 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8681 if (rc != VINF_SUCCESS)
8682 break;
8683 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8684 {
8685 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8686 rc = VINF_EM_RAW_INTERRUPT;
8687 break;
8688 }
8689 }
8690
8691 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8692 return rc;
8693}
8694
8695
8696/**
8697 * Single steps guest code using VT-x.
8698 *
8699 * @returns VBox status code.
8700 * @param pVM Pointer to the VM.
8701 * @param pVCpu Pointer to the VMCPU.
8702 * @param pCtx Pointer to the guest-CPU context.
8703 *
8704 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8705 */
8706static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8707{
8708 VMXTRANSIENT VmxTransient;
8709 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8710 int rc = VERR_INTERNAL_ERROR_5;
8711 uint32_t cLoops = 0;
8712 uint16_t uCsStart = pCtx->cs.Sel;
8713 uint64_t uRipStart = pCtx->rip;
8714
8715 for (;; cLoops++)
8716 {
8717 Assert(!HMR0SuspendPending());
8718 HMVMX_ASSERT_CPU_SAFE();
8719
8720 /* Preparatory work for running guest code, this may force us to return
8721 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8722 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8723 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8724 if (rc != VINF_SUCCESS)
8725 break;
8726
8727 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8728 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8729 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8730
8731 /* Restore any residual host-state and save any bits shared between host
8732 and guest into the guest-CPU state. Re-enables interrupts! */
8733 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8734
8735 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8736 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8737 {
8738 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8739 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8740 return rc;
8741 }
8742
8743 /* Handle the VM-exit. */
8744 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8745 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8746 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8747 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8748 HMVMX_START_EXIT_DISPATCH_PROF();
8749#ifdef HMVMX_USE_FUNCTION_TABLE
8750 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8751#else
8752 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8753#endif
8754 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8755 if (rc != VINF_SUCCESS)
8756 break;
8757 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8758 {
8759 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8760 rc = VINF_EM_RAW_INTERRUPT;
8761 break;
8762 }
8763
8764 /*
8765 * Did the RIP change, if so, consider it a single step.
8766 * Otherwise, make sure one of the TFs gets set.
8767 */
8768 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8769 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8770 AssertRCReturn(rc2, rc2);
8771 if ( pCtx->rip != uRipStart
8772 || pCtx->cs.Sel != uCsStart)
8773 {
8774 rc = VINF_EM_DBG_STEPPED;
8775 break;
8776 }
8777 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8778 }
8779
8780 /*
8781 * Clear the X86_EFL_TF if necessary.
8782 */
8783 if (pVCpu->hm.s.fClearTrapFlag)
8784 {
8785 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8786 AssertRCReturn(rc2, rc2);
8787 pVCpu->hm.s.fClearTrapFlag = false;
8788 pCtx->eflags.Bits.u1TF = 0;
8789 }
8790 /** @todo there seems to be issues with the resume flag when the monitor trap
8791 * flag is pending without being used. Seen early in bios init when
8792 * accessing APIC page in protected mode. */
8793
8794 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8795 return rc;
8796}
8797
8798
8799/**
8800 * Runs the guest code using VT-x.
8801 *
8802 * @returns VBox status code.
8803 * @param pVM Pointer to the VM.
8804 * @param pVCpu Pointer to the VMCPU.
8805 * @param pCtx Pointer to the guest-CPU context.
8806 */
8807VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8808{
8809 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8810 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8811 HMVMX_ASSERT_PREEMPT_SAFE();
8812
8813 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8814
8815 int rc;
8816 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8817 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8818 else
8819 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8820
8821 if (rc == VERR_EM_INTERPRETER)
8822 rc = VINF_EM_RAW_EMULATE_INSTR;
8823 else if (rc == VINF_EM_RESET)
8824 rc = VINF_EM_TRIPLE_FAULT;
8825
8826 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8827 if (RT_FAILURE(rc2))
8828 {
8829 pVCpu->hm.s.u32HMError = rc;
8830 rc = rc2;
8831 }
8832 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8833 return rc;
8834}
8835
8836
8837#ifndef HMVMX_USE_FUNCTION_TABLE
8838DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8839{
8840#ifdef DEBUG_ramshankar
8841# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8842# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8843#endif
8844 int rc;
8845 switch (rcReason)
8846 {
8847 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8848 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8849 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8850 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8851 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8852 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8853 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8854 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8855 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8856 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8857 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8858 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8859 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8860 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8861 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8862 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8863 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8864 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8865 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8866 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8867 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8868 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8869 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8870 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8871 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8872 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8873 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8874 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8875 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8876 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8877 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8878 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8879 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8880
8881 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8882 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8883 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8884 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8885 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8886 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8887 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8888 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8889 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8890
8891 case VMX_EXIT_VMCALL:
8892 case VMX_EXIT_VMCLEAR:
8893 case VMX_EXIT_VMLAUNCH:
8894 case VMX_EXIT_VMPTRLD:
8895 case VMX_EXIT_VMPTRST:
8896 case VMX_EXIT_VMREAD:
8897 case VMX_EXIT_VMRESUME:
8898 case VMX_EXIT_VMWRITE:
8899 case VMX_EXIT_VMXOFF:
8900 case VMX_EXIT_VMXON:
8901 case VMX_EXIT_INVEPT:
8902 case VMX_EXIT_INVVPID:
8903 case VMX_EXIT_VMFUNC:
8904 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8905 break;
8906 default:
8907 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8908 break;
8909 }
8910 return rc;
8911}
8912#endif
8913
8914#ifdef DEBUG
8915/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8916# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8917 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8918
8919# define HMVMX_ASSERT_PREEMPT_CPUID() \
8920 do \
8921 { \
8922 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8923 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8924 } while (0)
8925
8926# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8927 do { \
8928 AssertPtr(pVCpu); \
8929 AssertPtr(pMixedCtx); \
8930 AssertPtr(pVmxTransient); \
8931 Assert(pVmxTransient->fVMEntryFailed == false); \
8932 Assert(ASMIntAreEnabled()); \
8933 HMVMX_ASSERT_PREEMPT_SAFE(); \
8934 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8935 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8936 HMVMX_ASSERT_PREEMPT_SAFE(); \
8937 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8938 HMVMX_ASSERT_PREEMPT_CPUID(); \
8939 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8940 } while (0)
8941
8942# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8943 do { \
8944 Log4Func(("\n")); \
8945 } while (0)
8946#else /* Release builds */
8947# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8948 do { \
8949 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8950 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8951 } while (0)
8952# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8953#endif
8954
8955
8956/**
8957 * Advances the guest RIP after reading it from the VMCS.
8958 *
8959 * @returns VBox status code.
8960 * @param pVCpu Pointer to the VMCPU.
8961 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8962 * out-of-sync. Make sure to update the required fields
8963 * before using them.
8964 * @param pVmxTransient Pointer to the VMX transient structure.
8965 *
8966 * @remarks No-long-jump zone!!!
8967 */
8968DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8969{
8970 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8971 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8972 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8973 AssertRCReturn(rc, rc);
8974
8975 pMixedCtx->rip += pVmxTransient->cbInstr;
8976 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8977
8978 /*
8979 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8980 * pending debug exception field as it takes care of priority of events.
8981 *
8982 * See Intel spec. 32.2.1 "Debug Exceptions".
8983 */
8984 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
8985
8986 return rc;
8987}
8988
8989
8990/**
8991 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8992 * and update error record fields accordingly.
8993 *
8994 * @return VMX_IGS_* return codes.
8995 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8996 * wrong with the guest state.
8997 *
8998 * @param pVM Pointer to the VM.
8999 * @param pVCpu Pointer to the VMCPU.
9000 * @param pCtx Pointer to the guest-CPU state.
9001 *
9002 * @remarks This function assumes our cache of the VMCS controls
9003 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9004 */
9005static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9006{
9007#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9008#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9009 uError = (err); \
9010 break; \
9011 } else do { } while (0)
9012
9013 int rc;
9014 uint32_t uError = VMX_IGS_ERROR;
9015 uint32_t u32Val;
9016 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9017
9018 do
9019 {
9020 /*
9021 * CR0.
9022 */
9023 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9024 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9025 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9026 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9027 if (fUnrestrictedGuest)
9028 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9029
9030 uint32_t u32GuestCR0;
9031 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9032 AssertRCBreak(rc);
9033 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9034 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9035 if ( !fUnrestrictedGuest
9036 && (u32GuestCR0 & X86_CR0_PG)
9037 && !(u32GuestCR0 & X86_CR0_PE))
9038 {
9039 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9040 }
9041
9042 /*
9043 * CR4.
9044 */
9045 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9046 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9047
9048 uint32_t u32GuestCR4;
9049 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9050 AssertRCBreak(rc);
9051 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9052 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9053
9054 /*
9055 * IA32_DEBUGCTL MSR.
9056 */
9057 uint64_t u64Val;
9058 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9059 AssertRCBreak(rc);
9060 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9061 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9062 {
9063 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9064 }
9065 uint64_t u64DebugCtlMsr = u64Val;
9066
9067#ifdef VBOX_STRICT
9068 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9069 AssertRCBreak(rc);
9070 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9071#endif
9072 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9073
9074 /*
9075 * RIP and RFLAGS.
9076 */
9077 uint32_t u32Eflags;
9078#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9079 if (HMVMX_IS_64BIT_HOST_MODE())
9080 {
9081 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9082 AssertRCBreak(rc);
9083 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9084 if ( !fLongModeGuest
9085 || !pCtx->cs.Attr.n.u1Long)
9086 {
9087 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9088 }
9089 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9090 * must be identical if the "IA-32e mode guest" VM-entry
9091 * control is 1 and CS.L is 1. No check applies if the
9092 * CPU supports 64 linear-address bits. */
9093
9094 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9095 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9096 AssertRCBreak(rc);
9097 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9098 VMX_IGS_RFLAGS_RESERVED);
9099 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9100 u32Eflags = u64Val;
9101 }
9102 else
9103#endif
9104 {
9105 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9106 AssertRCBreak(rc);
9107 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9108 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9109 }
9110
9111 if ( fLongModeGuest
9112 || ( fUnrestrictedGuest
9113 && !(u32GuestCR0 & X86_CR0_PE)))
9114 {
9115 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9116 }
9117
9118 uint32_t u32EntryInfo;
9119 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9120 AssertRCBreak(rc);
9121 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9122 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9123 {
9124 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9125 }
9126
9127 /*
9128 * 64-bit checks.
9129 */
9130#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9131 if (HMVMX_IS_64BIT_HOST_MODE())
9132 {
9133 if ( fLongModeGuest
9134 && !fUnrestrictedGuest)
9135 {
9136 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9137 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9138 }
9139
9140 if ( !fLongModeGuest
9141 && (u32GuestCR4 & X86_CR4_PCIDE))
9142 {
9143 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9144 }
9145
9146 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9147 * 51:32 beyond the processor's physical-address width are 0. */
9148
9149 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9150 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9151 {
9152 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9153 }
9154
9155 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9156 AssertRCBreak(rc);
9157 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9158
9159 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9160 AssertRCBreak(rc);
9161 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9162 }
9163#endif
9164
9165 /*
9166 * PERF_GLOBAL MSR.
9167 */
9168 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9169 {
9170 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9171 AssertRCBreak(rc);
9172 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9173 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9174 }
9175
9176 /*
9177 * PAT MSR.
9178 */
9179 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9180 {
9181 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9182 AssertRCBreak(rc);
9183 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9184 for (unsigned i = 0; i < 8; i++)
9185 {
9186 uint8_t u8Val = (u64Val & 0x7);
9187 if ( u8Val != 0 /* UC */
9188 || u8Val != 1 /* WC */
9189 || u8Val != 4 /* WT */
9190 || u8Val != 5 /* WP */
9191 || u8Val != 6 /* WB */
9192 || u8Val != 7 /* UC- */)
9193 {
9194 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9195 }
9196 u64Val >>= 3;
9197 }
9198 }
9199
9200 /*
9201 * EFER MSR.
9202 */
9203 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9204 {
9205 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9206 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9207 AssertRCBreak(rc);
9208 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9209 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9210 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9211 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9212 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9213 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9214 VMX_IGS_EFER_LMA_PG_MISMATCH);
9215 }
9216
9217 /*
9218 * Segment registers.
9219 */
9220 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9221 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9222 if (!(u32Eflags & X86_EFL_VM))
9223 {
9224 /* CS */
9225 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9226 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9227 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9228 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9229 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9230 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9231 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9232 /* CS cannot be loaded with NULL in protected mode. */
9233 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9234 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9235 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9236 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9237 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9238 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9239 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9240 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9241 else
9242 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9243
9244 /* SS */
9245 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9246 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9247 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9248 if ( !(pCtx->cr0 & X86_CR0_PE)
9249 || pCtx->cs.Attr.n.u4Type == 3)
9250 {
9251 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9252 }
9253 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9254 {
9255 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9256 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9257 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9258 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9259 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9260 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9261 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9262 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9263 }
9264
9265 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9266 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9267 {
9268 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9269 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9270 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9271 || pCtx->ds.Attr.n.u4Type > 11
9272 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9273 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9274 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9275 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9276 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9277 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9278 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9279 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9280 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9281 }
9282 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9283 {
9284 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9285 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9286 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9287 || pCtx->es.Attr.n.u4Type > 11
9288 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9289 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9290 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9291 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9292 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9293 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9294 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9295 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9296 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9297 }
9298 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9299 {
9300 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9301 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9302 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9303 || pCtx->fs.Attr.n.u4Type > 11
9304 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9305 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9306 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9307 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9308 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9309 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9310 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9311 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9312 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9313 }
9314 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9315 {
9316 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9317 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9318 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9319 || pCtx->gs.Attr.n.u4Type > 11
9320 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9321 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9322 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9323 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9324 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9325 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9326 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9327 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9328 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9329 }
9330 /* 64-bit capable CPUs. */
9331#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9332 if (HMVMX_IS_64BIT_HOST_MODE())
9333 {
9334 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9335 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9336 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9337 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9338 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9339 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9340 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9341 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9342 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9343 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9344 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9345 }
9346#endif
9347 }
9348 else
9349 {
9350 /* V86 mode checks. */
9351 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9352 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9353 {
9354 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9355 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9356 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9357 }
9358 else
9359 {
9360 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9361 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9362 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9363 }
9364
9365 /* CS */
9366 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9367 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9368 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9369 /* SS */
9370 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9371 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9372 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9373 /* DS */
9374 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9375 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9376 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9377 /* ES */
9378 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9379 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9380 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9381 /* FS */
9382 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9383 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9384 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9385 /* GS */
9386 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9387 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9388 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9389 /* 64-bit capable CPUs. */
9390#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9391 if (HMVMX_IS_64BIT_HOST_MODE())
9392 {
9393 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9394 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9395 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9396 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9397 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9398 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9399 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9400 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9401 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9402 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9403 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9404 }
9405#endif
9406 }
9407
9408 /*
9409 * TR.
9410 */
9411 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9412 /* 64-bit capable CPUs. */
9413#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9414 if (HMVMX_IS_64BIT_HOST_MODE())
9415 {
9416 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9417 }
9418#endif
9419 if (fLongModeGuest)
9420 {
9421 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9422 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9423 }
9424 else
9425 {
9426 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9427 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9428 VMX_IGS_TR_ATTR_TYPE_INVALID);
9429 }
9430 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9431 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9432 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9433 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9434 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9435 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9436 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9437 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9438
9439 /*
9440 * GDTR and IDTR.
9441 */
9442#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9443 if (HMVMX_IS_64BIT_HOST_MODE())
9444 {
9445 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9446 AssertRCBreak(rc);
9447 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9448
9449 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9450 AssertRCBreak(rc);
9451 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9452 }
9453#endif
9454
9455 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9456 AssertRCBreak(rc);
9457 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9458
9459 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9460 AssertRCBreak(rc);
9461 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9462
9463 /*
9464 * Guest Non-Register State.
9465 */
9466 /* Activity State. */
9467 uint32_t u32ActivityState;
9468 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9469 AssertRCBreak(rc);
9470 HMVMX_CHECK_BREAK( !u32ActivityState
9471 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9472 VMX_IGS_ACTIVITY_STATE_INVALID);
9473 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9474 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9475 uint32_t u32IntrState;
9476 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9477 AssertRCBreak(rc);
9478 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9479 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9480 {
9481 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9482 }
9483
9484 /** @todo Activity state and injecting interrupts. Left as a todo since we
9485 * currently don't use activity states but ACTIVE. */
9486
9487 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9488 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9489
9490 /* Guest interruptibility-state. */
9491 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9492 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9493 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9494 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9495 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9496 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9497 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9498 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9499 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9500 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9501 {
9502 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9503 {
9504 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9505 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9506 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9507 }
9508 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9509 {
9510 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9511 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9512 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9513 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9514 }
9515 }
9516 /** @todo Assumes the processor is not in SMM. */
9517 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9518 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9519 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9520 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9521 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9522 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9523 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9524 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9525 {
9526 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9527 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9528 }
9529
9530 /* Pending debug exceptions. */
9531 if (HMVMX_IS_64BIT_HOST_MODE())
9532 {
9533 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9534 AssertRCBreak(rc);
9535 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9536 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9537 u32Val = u64Val; /* For pending debug exceptions checks below. */
9538 }
9539 else
9540 {
9541 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9542 AssertRCBreak(rc);
9543 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9544 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9545 }
9546
9547 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9548 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9549 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9550 {
9551 if ( (u32Eflags & X86_EFL_TF)
9552 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9553 {
9554 /* Bit 14 is PendingDebug.BS. */
9555 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9556 }
9557 if ( !(u32Eflags & X86_EFL_TF)
9558 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9559 {
9560 /* Bit 14 is PendingDebug.BS. */
9561 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9562 }
9563 }
9564
9565 /* VMCS link pointer. */
9566 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9567 AssertRCBreak(rc);
9568 if (u64Val != UINT64_C(0xffffffffffffffff))
9569 {
9570 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9571 /** @todo Bits beyond the processor's physical-address width MBZ. */
9572 /** @todo 32-bit located in memory referenced by value of this field (as a
9573 * physical address) must contain the processor's VMCS revision ID. */
9574 /** @todo SMM checks. */
9575 }
9576
9577 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9578 * not using Nested Paging? */
9579 if ( pVM->hm.s.fNestedPaging
9580 && !fLongModeGuest
9581 && CPUMIsGuestInPAEModeEx(pCtx))
9582 {
9583 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9584 AssertRCBreak(rc);
9585 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9586
9587 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9588 AssertRCBreak(rc);
9589 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9590
9591 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9592 AssertRCBreak(rc);
9593 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9594
9595 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9596 AssertRCBreak(rc);
9597 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9598 }
9599
9600 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9601 if (uError == VMX_IGS_ERROR)
9602 uError = VMX_IGS_REASON_NOT_FOUND;
9603 } while (0);
9604
9605 pVCpu->hm.s.u32HMError = uError;
9606 return uError;
9607
9608#undef HMVMX_ERROR_BREAK
9609#undef HMVMX_CHECK_BREAK
9610}
9611
9612/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9613/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9614/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9615
9616/** @name VM-exit handlers.
9617 * @{
9618 */
9619
9620/**
9621 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9622 */
9623HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9624{
9625 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9626 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9627 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9628 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9629 return VINF_SUCCESS;
9630 return VINF_EM_RAW_INTERRUPT;
9631}
9632
9633
9634/**
9635 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9636 */
9637HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9638{
9639 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9640 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9641
9642 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9643 AssertRCReturn(rc, rc);
9644
9645 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9646 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9647 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9648 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9649
9650 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9651 {
9652 /*
9653 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9654 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9655 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9656 *
9657 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9658 */
9659 VMXDispatchHostNmi();
9660 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9661 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9662 return VINF_SUCCESS;
9663 }
9664
9665 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9666 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9667 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9668 {
9669 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9670 return VINF_SUCCESS;
9671 }
9672 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9673 {
9674 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9675 return rc;
9676 }
9677
9678 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9679 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9680 switch (uIntType)
9681 {
9682 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9683 Assert(uVector == X86_XCPT_DB);
9684 /* no break */
9685 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9686 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9687 /* no break */
9688 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9689 {
9690 switch (uVector)
9691 {
9692 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9693 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9694 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9695 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9696 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9697 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9698#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9699 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9700 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9701 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9702 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9703 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9704 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9705 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9706 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9707 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9708 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9709 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9710 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9711#endif
9712 default:
9713 {
9714 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9715 AssertRCReturn(rc, rc);
9716
9717 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9718 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9719 {
9720 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9721 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9722 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9723
9724 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9725 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9726 AssertRCReturn(rc, rc);
9727 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9728 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9729 0 /* GCPtrFaultAddress */);
9730 AssertRCReturn(rc, rc);
9731 }
9732 else
9733 {
9734 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9735 pVCpu->hm.s.u32HMError = uVector;
9736 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9737 }
9738 break;
9739 }
9740 }
9741 break;
9742 }
9743
9744 default:
9745 {
9746 pVCpu->hm.s.u32HMError = uExitIntInfo;
9747 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9748 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9749 break;
9750 }
9751 }
9752 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9753 return rc;
9754}
9755
9756
9757/**
9758 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9759 */
9760HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9761{
9762 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9763
9764 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9765 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9766
9767 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9768 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9769 return VINF_SUCCESS;
9770}
9771
9772
9773/**
9774 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9775 */
9776HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9777{
9778 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9779 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9780 HMVMX_RETURN_UNEXPECTED_EXIT();
9781}
9782
9783
9784/**
9785 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9786 */
9787HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9788{
9789 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9790 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9791 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9792}
9793
9794
9795/**
9796 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9797 */
9798HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9799{
9800 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9801 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9802 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9803}
9804
9805
9806/**
9807 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9808 */
9809HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9810{
9811 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9812 PVM pVM = pVCpu->CTX_SUFF(pVM);
9813 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9814 if (RT_LIKELY(rc == VINF_SUCCESS))
9815 {
9816 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9817 Assert(pVmxTransient->cbInstr == 2);
9818 }
9819 else
9820 {
9821 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9822 rc = VERR_EM_INTERPRETER;
9823 }
9824 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9825 return rc;
9826}
9827
9828
9829/**
9830 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9831 */
9832HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9833{
9834 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9835 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9836 AssertRCReturn(rc, rc);
9837
9838 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9839 return VINF_EM_RAW_EMULATE_INSTR;
9840
9841 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9842 HMVMX_RETURN_UNEXPECTED_EXIT();
9843}
9844
9845
9846/**
9847 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9848 */
9849HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9850{
9851 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9852 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9853 AssertRCReturn(rc, rc);
9854
9855 PVM pVM = pVCpu->CTX_SUFF(pVM);
9856 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9857 if (RT_LIKELY(rc == VINF_SUCCESS))
9858 {
9859 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9860 Assert(pVmxTransient->cbInstr == 2);
9861 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9862 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9863 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9864 }
9865 else
9866 {
9867 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9868 rc = VERR_EM_INTERPRETER;
9869 }
9870 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9871 return rc;
9872}
9873
9874
9875/**
9876 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9877 */
9878HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9879{
9880 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9881 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9882 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9883 AssertRCReturn(rc, rc);
9884
9885 PVM pVM = pVCpu->CTX_SUFF(pVM);
9886 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9887 if (RT_LIKELY(rc == VINF_SUCCESS))
9888 {
9889 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9890 Assert(pVmxTransient->cbInstr == 3);
9891 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9892 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9893 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9894 }
9895 else
9896 {
9897 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9898 rc = VERR_EM_INTERPRETER;
9899 }
9900 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9901 return rc;
9902}
9903
9904
9905/**
9906 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9907 */
9908HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9909{
9910 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9911 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9912 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9913 AssertRCReturn(rc, rc);
9914
9915 PVM pVM = pVCpu->CTX_SUFF(pVM);
9916 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9917 if (RT_LIKELY(rc == VINF_SUCCESS))
9918 {
9919 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9920 Assert(pVmxTransient->cbInstr == 2);
9921 }
9922 else
9923 {
9924 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9925 rc = VERR_EM_INTERPRETER;
9926 }
9927 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9928 return rc;
9929}
9930
9931
9932/**
9933 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9934 */
9935HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9936{
9937 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9938 PVM pVM = pVCpu->CTX_SUFF(pVM);
9939 Assert(!pVM->hm.s.fNestedPaging);
9940
9941 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9942 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9943 AssertRCReturn(rc, rc);
9944
9945 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9946 rc = VBOXSTRICTRC_VAL(rc2);
9947 if (RT_LIKELY(rc == VINF_SUCCESS))
9948 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9949 else
9950 {
9951 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9952 pVmxTransient->uExitQualification, rc));
9953 }
9954 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9955 return rc;
9956}
9957
9958
9959/**
9960 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9961 */
9962HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9963{
9964 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9965 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9966 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9967 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9968 AssertRCReturn(rc, rc);
9969
9970 PVM pVM = pVCpu->CTX_SUFF(pVM);
9971 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9972 if (RT_LIKELY(rc == VINF_SUCCESS))
9973 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9974 else
9975 {
9976 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9977 rc = VERR_EM_INTERPRETER;
9978 }
9979 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9980 return rc;
9981}
9982
9983
9984/**
9985 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9986 */
9987HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9988{
9989 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9990 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9991 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9992 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9993 AssertRCReturn(rc, rc);
9994
9995 PVM pVM = pVCpu->CTX_SUFF(pVM);
9996 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9997 rc = VBOXSTRICTRC_VAL(rc2);
9998 if (RT_LIKELY( rc == VINF_SUCCESS
9999 || rc == VINF_EM_HALT))
10000 {
10001 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10002 AssertRCReturn(rc3, rc3);
10003
10004 if ( rc == VINF_EM_HALT
10005 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10006 {
10007 rc = VINF_SUCCESS;
10008 }
10009 }
10010 else
10011 {
10012 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10013 rc = VERR_EM_INTERPRETER;
10014 }
10015 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10016 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10017 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10018 return rc;
10019}
10020
10021
10022/**
10023 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10024 */
10025HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10026{
10027 /*
10028 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10029 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10030 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10031 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10032 */
10033 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10034 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10035 HMVMX_RETURN_UNEXPECTED_EXIT();
10036}
10037
10038
10039/**
10040 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10041 */
10042HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10043{
10044 /*
10045 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10046 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10047 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10048 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10049 */
10050 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10051 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10052 HMVMX_RETURN_UNEXPECTED_EXIT();
10053}
10054
10055
10056/**
10057 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10058 */
10059HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10060{
10061 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10062 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10063 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10064 HMVMX_RETURN_UNEXPECTED_EXIT();
10065}
10066
10067
10068/**
10069 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10070 */
10071HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10072{
10073 /*
10074 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10075 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10076 * See Intel spec. 25.3 "Other Causes of VM-exits".
10077 */
10078 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10079 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10080 HMVMX_RETURN_UNEXPECTED_EXIT();
10081}
10082
10083
10084/**
10085 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10086 * VM-exit.
10087 */
10088HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10089{
10090 /*
10091 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10092 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10093 *
10094 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10095 * See Intel spec. "23.8 Restrictions on VMX operation".
10096 */
10097 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10098 return VINF_SUCCESS;
10099}
10100
10101
10102/**
10103 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10104 * VM-exit.
10105 */
10106HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10107{
10108 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10109 return VINF_EM_RESET;
10110}
10111
10112
10113/**
10114 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10115 */
10116HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10117{
10118 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10119 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10120 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10121 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10122 AssertRCReturn(rc, rc);
10123
10124 pMixedCtx->rip++;
10125 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10126 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10127 rc = VINF_SUCCESS;
10128 else
10129 rc = VINF_EM_HALT;
10130
10131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10132 return rc;
10133}
10134
10135
10136/**
10137 * VM-exit handler for instructions that result in a #UD exception delivered to
10138 * the guest.
10139 */
10140HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10141{
10142 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10143 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10144 return VINF_SUCCESS;
10145}
10146
10147
10148/**
10149 * VM-exit handler for expiry of the VMX preemption timer.
10150 */
10151HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10152{
10153 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10154
10155 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10156 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10157
10158 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10159 PVM pVM = pVCpu->CTX_SUFF(pVM);
10160 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10161 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10162 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10163}
10164
10165
10166/**
10167 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10168 */
10169HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10170{
10171 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10172
10173 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10174 /** @todo check if XSETBV is supported by the recompiler. */
10175 return VERR_EM_INTERPRETER;
10176}
10177
10178
10179/**
10180 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10181 */
10182HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10183{
10184 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10185
10186 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10187 /** @todo implement EMInterpretInvpcid() */
10188 return VERR_EM_INTERPRETER;
10189}
10190
10191
10192/**
10193 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10194 * Error VM-exit.
10195 */
10196HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10197{
10198 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10199 AssertRCReturn(rc, rc);
10200
10201 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10202 AssertRCReturn(rc, rc);
10203
10204 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10205 NOREF(uInvalidReason);
10206
10207#ifdef VBOX_STRICT
10208 uint32_t uIntrState;
10209 HMVMXHCUINTREG uHCReg;
10210 uint64_t u64Val;
10211 uint32_t u32Val;
10212
10213 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10214 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10215 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10216 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10217 AssertRCReturn(rc, rc);
10218
10219 Log4(("uInvalidReason %u\n", uInvalidReason));
10220 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10221 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10222 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10223 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10224
10225 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10226 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10227 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10228 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10229 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10230 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10231 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10232 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10233 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10234 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10235 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10236 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10237#else
10238 NOREF(pVmxTransient);
10239#endif
10240
10241 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10242 return VERR_VMX_INVALID_GUEST_STATE;
10243}
10244
10245
10246/**
10247 * VM-exit handler for VM-entry failure due to an MSR-load
10248 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10249 */
10250HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10251{
10252 NOREF(pVmxTransient);
10253 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10254 HMVMX_RETURN_UNEXPECTED_EXIT();
10255}
10256
10257
10258/**
10259 * VM-exit handler for VM-entry failure due to a machine-check event
10260 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10261 */
10262HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10263{
10264 NOREF(pVmxTransient);
10265 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10266 HMVMX_RETURN_UNEXPECTED_EXIT();
10267}
10268
10269
10270/**
10271 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10272 * theory.
10273 */
10274HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10275{
10276 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10277 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10278 return VERR_VMX_UNDEFINED_EXIT_CODE;
10279}
10280
10281
10282/**
10283 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10284 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10285 * Conditional VM-exit.
10286 */
10287HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10288{
10289 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10290
10291 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10292 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10293 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10294 return VERR_EM_INTERPRETER;
10295 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10296 HMVMX_RETURN_UNEXPECTED_EXIT();
10297}
10298
10299
10300/**
10301 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10302 */
10303HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10304{
10305 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10306
10307 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10308 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10309 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10310 return VERR_EM_INTERPRETER;
10311 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10312 HMVMX_RETURN_UNEXPECTED_EXIT();
10313}
10314
10315
10316/**
10317 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10318 */
10319HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10320{
10321 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10322
10323 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10324 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10325 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10326 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10327 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10328 {
10329 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10330 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10331 }
10332 AssertRCReturn(rc, rc);
10333 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10334
10335#ifdef VBOX_STRICT
10336 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10337 {
10338 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10339 && pMixedCtx->ecx != MSR_K6_EFER)
10340 {
10341 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10342 HMVMX_RETURN_UNEXPECTED_EXIT();
10343 }
10344# if HC_ARCH_BITS == 64
10345 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10346 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10347 {
10348 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10349 HMVMX_RETURN_UNEXPECTED_EXIT();
10350 }
10351# endif
10352 }
10353#endif
10354
10355 PVM pVM = pVCpu->CTX_SUFF(pVM);
10356 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10357 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10358 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10359 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10360
10361 if (RT_LIKELY(rc == VINF_SUCCESS))
10362 {
10363 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10364 Assert(pVmxTransient->cbInstr == 2);
10365 }
10366 return rc;
10367}
10368
10369
10370/**
10371 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10372 */
10373HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10374{
10375 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10376 PVM pVM = pVCpu->CTX_SUFF(pVM);
10377 int rc = VINF_SUCCESS;
10378
10379 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10380 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10381 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10382 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10383 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10384 {
10385 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10386 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10387 }
10388 AssertRCReturn(rc, rc);
10389 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10390
10391 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10392 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10393 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10394
10395 if (RT_LIKELY(rc == VINF_SUCCESS))
10396 {
10397 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10398
10399 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10400 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10401 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10402 {
10403 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10404 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10405 EMInterpretWrmsr() changes it. */
10406 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10407 }
10408 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10409 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10410 else if (pMixedCtx->ecx == MSR_K6_EFER)
10411 {
10412 /*
10413 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10414 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10415 * the other bits as well, SCE and NXE. See @bugref{7368}.
10416 */
10417 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10418 }
10419
10420 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10421 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10422 {
10423 switch (pMixedCtx->ecx)
10424 {
10425 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10426 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10427 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10428 case MSR_K8_FS_BASE: /* no break */
10429 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10430 case MSR_K6_EFER: /* already handled above */ break;
10431 default:
10432 {
10433 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10434 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10435#if HC_ARCH_BITS == 64
10436 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10437 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10438#endif
10439 break;
10440 }
10441 }
10442 }
10443#ifdef VBOX_STRICT
10444 else
10445 {
10446 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10447 switch (pMixedCtx->ecx)
10448 {
10449 case MSR_IA32_SYSENTER_CS:
10450 case MSR_IA32_SYSENTER_EIP:
10451 case MSR_IA32_SYSENTER_ESP:
10452 case MSR_K8_FS_BASE:
10453 case MSR_K8_GS_BASE:
10454 {
10455 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10456 HMVMX_RETURN_UNEXPECTED_EXIT();
10457 }
10458
10459 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10460 default:
10461 {
10462 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10463 {
10464 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10465 if (pMixedCtx->ecx != MSR_K6_EFER)
10466 {
10467 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10468 pMixedCtx->ecx));
10469 HMVMX_RETURN_UNEXPECTED_EXIT();
10470 }
10471 }
10472
10473#if HC_ARCH_BITS == 64
10474 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10475 {
10476 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10477 HMVMX_RETURN_UNEXPECTED_EXIT();
10478 }
10479#endif
10480 break;
10481 }
10482 }
10483 }
10484#endif /* VBOX_STRICT */
10485 }
10486 return rc;
10487}
10488
10489
10490/**
10491 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10492 */
10493HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10494{
10495 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10496
10497 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10498 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10499 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10500 return VERR_EM_INTERPRETER;
10501 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10502 HMVMX_RETURN_UNEXPECTED_EXIT();
10503}
10504
10505
10506/**
10507 * VM-exit handler for when the TPR value is lowered below the specified
10508 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10509 */
10510HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10511{
10512 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10513 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10514
10515 /*
10516 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10517 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10518 * resume guest execution.
10519 */
10520 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10521 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10522 return VINF_SUCCESS;
10523}
10524
10525
10526/**
10527 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10528 * VM-exit.
10529 *
10530 * @retval VINF_SUCCESS when guest execution can continue.
10531 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10532 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10533 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10534 * recompiler.
10535 */
10536HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10537{
10538 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10539 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10540 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10541 AssertRCReturn(rc, rc);
10542
10543 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10544 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10545 PVM pVM = pVCpu->CTX_SUFF(pVM);
10546 switch (uAccessType)
10547 {
10548 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10549 {
10550#if 0
10551 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10552 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10553#else
10554 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10555 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10556 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10557#endif
10558 AssertRCReturn(rc, rc);
10559
10560 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10561 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10562 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10563 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10564
10565 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10566 {
10567 case 0: /* CR0 */
10568 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10569 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10570 break;
10571 case 2: /* CR2 */
10572 /* Nothing to do here, CR2 it's not part of the VMCS. */
10573 break;
10574 case 3: /* CR3 */
10575 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10576 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10577 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10578 break;
10579 case 4: /* CR4 */
10580 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10581 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10582 break;
10583 case 8: /* CR8 */
10584 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10585 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10586 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10587 break;
10588 default:
10589 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10590 break;
10591 }
10592
10593 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10594 break;
10595 }
10596
10597 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10598 {
10599 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10600 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10601 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10602 AssertRCReturn(rc, rc);
10603 Assert( !pVM->hm.s.fNestedPaging
10604 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10605 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10606
10607 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10608 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10609 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10610
10611 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10612 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10613 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10614 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10615 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10616 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10617 break;
10618 }
10619
10620 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10621 {
10622 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10623 AssertRCReturn(rc, rc);
10624 rc = EMInterpretCLTS(pVM, pVCpu);
10625 AssertRCReturn(rc, rc);
10626 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10627 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10628 Log4(("CRX CLTS write rc=%d\n", rc));
10629 break;
10630 }
10631
10632 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10633 {
10634 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10635 AssertRCReturn(rc, rc);
10636 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10637 if (RT_LIKELY(rc == VINF_SUCCESS))
10638 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10639 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10640 Log4(("CRX LMSW write rc=%d\n", rc));
10641 break;
10642 }
10643
10644 default:
10645 {
10646 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10647 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10648 }
10649 }
10650
10651 /* Validate possible error codes. */
10652 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10653 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10654 if (RT_SUCCESS(rc))
10655 {
10656 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10657 AssertRCReturn(rc2, rc2);
10658 }
10659
10660 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10661 return rc;
10662}
10663
10664
10665/**
10666 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10667 * VM-exit.
10668 */
10669HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10670{
10671 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10672 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10673
10674 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10675 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10676 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10677 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10678 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10679 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10680 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10681 AssertRCReturn(rc2, rc2);
10682
10683 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10684 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10685 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10686 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10687 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10688 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10689 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10690 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10691
10692 /* I/O operation lookup arrays. */
10693 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10694 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10695
10696 VBOXSTRICTRC rcStrict;
10697 const uint32_t cbValue = s_aIOSizes[uIOWidth];
10698 const uint32_t cbInstr = pVmxTransient->cbInstr;
10699 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10700 PVM pVM = pVCpu->CTX_SUFF(pVM);
10701 if (fIOString)
10702 {
10703#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10704 /*
10705 * INS/OUTS - I/O String instruction.
10706 *
10707 * Use instruction-information if available, otherwise fall back on
10708 * interpreting the instruction.
10709 */
10710 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10711 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10712 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10713 {
10714 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10715 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10716 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10717 AssertRCReturn(rc2, rc2);
10718 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10719 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10720 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10721 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10722 if (fIOWrite)
10723 {
10724 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10725 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10726 }
10727 else
10728 {
10729 /*
10730 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10731 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10732 * See Intel Instruction spec. for "INS".
10733 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10734 */
10735 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10736 }
10737 }
10738 else
10739 {
10740 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10741 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10742 AssertRCReturn(rc2, rc2);
10743 rcStrict = IEMExecOne(pVCpu);
10744 }
10745 /** @todo IEM needs to be setting these flags somehow. */
10746 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10747 fUpdateRipAlready = true;
10748#else
10749 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10750 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10751 if (RT_SUCCESS(rcStrict))
10752 {
10753 if (fIOWrite)
10754 {
10755 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10756 (DISCPUMODE)pDis->uAddrMode, cbValue);
10757 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10758 }
10759 else
10760 {
10761 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10762 (DISCPUMODE)pDis->uAddrMode, cbValue);
10763 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10764 }
10765 }
10766 else
10767 {
10768 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10769 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10770 }
10771#endif
10772 }
10773 else
10774 {
10775 /*
10776 * IN/OUT - I/O instruction.
10777 */
10778 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10779 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10780 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10781 if (fIOWrite)
10782 {
10783 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10784 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10785 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10786 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10787 }
10788 else
10789 {
10790 uint32_t u32Result = 0;
10791 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10792 if (IOM_SUCCESS(rcStrict))
10793 {
10794 /* Save result of I/O IN instr. in AL/AX/EAX. */
10795 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10796 }
10797 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10798 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10800 }
10801 }
10802
10803 if (IOM_SUCCESS(rcStrict))
10804 {
10805 if (!fUpdateRipAlready)
10806 {
10807 pMixedCtx->rip += cbInstr;
10808 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10809 }
10810
10811 /*
10812 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10813 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10814 */
10815 if (fIOString)
10816 {
10817 /** @todo Single-step for INS/OUTS with REP prefix? */
10818 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10819 }
10820 else if (fStepping)
10821 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10822
10823 /*
10824 * If any I/O breakpoints are armed, we need to check if one triggered
10825 * and take appropriate action.
10826 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10827 */
10828 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10829 AssertRCReturn(rc2, rc2);
10830
10831 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10832 * execution engines about whether hyper BPs and such are pending. */
10833 uint32_t const uDr7 = pMixedCtx->dr[7];
10834 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10835 && X86_DR7_ANY_RW_IO(uDr7)
10836 && (pMixedCtx->cr4 & X86_CR4_DE))
10837 || DBGFBpIsHwIoArmed(pVM)))
10838 {
10839 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10840
10841 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10842 VMMRZCallRing3Disable(pVCpu);
10843 HM_DISABLE_PREEMPT_IF_NEEDED();
10844
10845 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10846
10847 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10848 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10849 {
10850 /* Raise #DB. */
10851 if (fIsGuestDbgActive)
10852 ASMSetDR6(pMixedCtx->dr[6]);
10853 if (pMixedCtx->dr[7] != uDr7)
10854 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10855
10856 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10857 }
10858 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10859 else if ( rcStrict2 != VINF_SUCCESS
10860 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10861 rcStrict = rcStrict2;
10862
10863 HM_RESTORE_PREEMPT_IF_NEEDED();
10864 VMMRZCallRing3Enable(pVCpu);
10865 }
10866 }
10867
10868#ifdef DEBUG
10869 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10870 Assert(!fIOWrite);
10871 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10872 Assert(fIOWrite);
10873 else
10874 {
10875 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10876 * statuses, that the VMM device and some others may return. See
10877 * IOM_SUCCESS() for guidance. */
10878 AssertMsg( RT_FAILURE(rcStrict)
10879 || rcStrict == VINF_SUCCESS
10880 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10881 || rcStrict == VINF_EM_DBG_BREAKPOINT
10882 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10883 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10884 }
10885#endif
10886
10887 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10888 return VBOXSTRICTRC_TODO(rcStrict);
10889}
10890
10891
10892/**
10893 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10894 * VM-exit.
10895 */
10896HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10897{
10898 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10899
10900 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10901 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10902 AssertRCReturn(rc, rc);
10903 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10904 {
10905 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10906 AssertRCReturn(rc, rc);
10907 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10908 {
10909 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10910
10911 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10912 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10913
10914 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10915 Assert(!pVCpu->hm.s.Event.fPending);
10916 pVCpu->hm.s.Event.fPending = true;
10917 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10918 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10919 AssertRCReturn(rc, rc);
10920 if (fErrorCodeValid)
10921 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10922 else
10923 pVCpu->hm.s.Event.u32ErrCode = 0;
10924 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10925 && uVector == X86_XCPT_PF)
10926 {
10927 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10928 }
10929
10930 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10931 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10932 return VINF_EM_RAW_INJECT_TRPM_EVENT;
10933 }
10934 }
10935
10936 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10937 * emulation. */
10938 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10939 return VERR_EM_INTERPRETER;
10940}
10941
10942
10943/**
10944 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10945 */
10946HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10947{
10948 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10949 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10950 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10951 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10952 AssertRCReturn(rc, rc);
10953 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10954 return VINF_EM_DBG_STEPPED;
10955}
10956
10957
10958/**
10959 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10960 */
10961HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10962{
10963 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10964
10965 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10966 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10967 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10968 return VINF_SUCCESS;
10969 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10970 return rc;
10971
10972#if 0
10973 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10974 * just sync the whole thing. */
10975 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10976#else
10977 /* Aggressive state sync. for now. */
10978 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10979 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10980 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10981#endif
10982 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10983 AssertRCReturn(rc, rc);
10984
10985 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10986 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10987 switch (uAccessType)
10988 {
10989 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10990 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10991 {
10992 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10993 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10994 {
10995 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10996 }
10997
10998 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10999 GCPhys &= PAGE_BASE_GC_MASK;
11000 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11001 PVM pVM = pVCpu->CTX_SUFF(pVM);
11002 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11003 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11004
11005 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11006 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11007 CPUMCTX2CORE(pMixedCtx), GCPhys);
11008 rc = VBOXSTRICTRC_VAL(rc2);
11009 Log4(("ApicAccess rc=%d\n", rc));
11010 if ( rc == VINF_SUCCESS
11011 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11012 || rc == VERR_PAGE_NOT_PRESENT)
11013 {
11014 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11015 | HM_CHANGED_GUEST_RSP
11016 | HM_CHANGED_GUEST_RFLAGS
11017 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11018 rc = VINF_SUCCESS;
11019 }
11020 break;
11021 }
11022
11023 default:
11024 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11025 rc = VINF_EM_RAW_EMULATE_INSTR;
11026 break;
11027 }
11028
11029 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11030 return rc;
11031}
11032
11033
11034/**
11035 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11036 * VM-exit.
11037 */
11038HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11039{
11040 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11041
11042 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11043 if (pVmxTransient->fWasGuestDebugStateActive)
11044 {
11045 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11046 HMVMX_RETURN_UNEXPECTED_EXIT();
11047 }
11048
11049 int rc = VERR_INTERNAL_ERROR_5;
11050 if ( !DBGFIsStepping(pVCpu)
11051 && !pVCpu->hm.s.fSingleInstruction
11052 && !pVmxTransient->fWasHyperDebugStateActive)
11053 {
11054 /* Don't intercept MOV DRx and #DB any more. */
11055 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11056 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11057 AssertRCReturn(rc, rc);
11058
11059 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11060 {
11061#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11062 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11063 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11064 AssertRCReturn(rc, rc);
11065#endif
11066 }
11067
11068 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11069 VMMRZCallRing3Disable(pVCpu);
11070 HM_DISABLE_PREEMPT_IF_NEEDED();
11071
11072 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11073 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11074 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11075
11076 HM_RESTORE_PREEMPT_IF_NEEDED();
11077 VMMRZCallRing3Enable(pVCpu);
11078
11079#ifdef VBOX_WITH_STATISTICS
11080 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11081 AssertRCReturn(rc, rc);
11082 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11083 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11084 else
11085 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11086#endif
11087 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11088 return VINF_SUCCESS;
11089 }
11090
11091 /*
11092 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11093 * Update the segment registers and DR7 from the CPU.
11094 */
11095 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11096 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11097 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11098 AssertRCReturn(rc, rc);
11099 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11100
11101 PVM pVM = pVCpu->CTX_SUFF(pVM);
11102 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11103 {
11104 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11105 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11106 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11107 if (RT_SUCCESS(rc))
11108 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11109 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11110 }
11111 else
11112 {
11113 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11114 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11115 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11116 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11117 }
11118
11119 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11120 if (RT_SUCCESS(rc))
11121 {
11122 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11123 AssertRCReturn(rc2, rc2);
11124 }
11125 return rc;
11126}
11127
11128
11129/**
11130 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11131 * Conditional VM-exit.
11132 */
11133HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11134{
11135 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11136 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11137
11138 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11139 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11140 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11141 return VINF_SUCCESS;
11142 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11143 return rc;
11144
11145 RTGCPHYS GCPhys = 0;
11146 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11147
11148#if 0
11149 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11150#else
11151 /* Aggressive state sync. for now. */
11152 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11153 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11154 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11155#endif
11156 AssertRCReturn(rc, rc);
11157
11158 /*
11159 * If we succeed, resume guest execution.
11160 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11161 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11162 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11163 * weird case. See @bugref{6043}.
11164 */
11165 PVM pVM = pVCpu->CTX_SUFF(pVM);
11166 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11167 rc = VBOXSTRICTRC_VAL(rc2);
11168 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11169 if ( rc == VINF_SUCCESS
11170 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11171 || rc == VERR_PAGE_NOT_PRESENT)
11172 {
11173 /* Successfully handled MMIO operation. */
11174 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11175 | HM_CHANGED_GUEST_RSP
11176 | HM_CHANGED_GUEST_RFLAGS
11177 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11178 rc = VINF_SUCCESS;
11179 }
11180 return rc;
11181}
11182
11183
11184/**
11185 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11186 * VM-exit.
11187 */
11188HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11189{
11190 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11191 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11192
11193 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11194 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11195 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11196 return VINF_SUCCESS;
11197 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11198 return rc;
11199
11200 RTGCPHYS GCPhys = 0;
11201 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11202 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11203#if 0
11204 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11205#else
11206 /* Aggressive state sync. for now. */
11207 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11208 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11209 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11210#endif
11211 AssertRCReturn(rc, rc);
11212
11213 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11214 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11215
11216 RTGCUINT uErrorCode = 0;
11217 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11218 uErrorCode |= X86_TRAP_PF_ID;
11219 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11220 uErrorCode |= X86_TRAP_PF_RW;
11221 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11222 uErrorCode |= X86_TRAP_PF_P;
11223
11224 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11225
11226 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11227 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11228
11229 /* Handle the pagefault trap for the nested shadow table. */
11230 PVM pVM = pVCpu->CTX_SUFF(pVM);
11231 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11232 TRPMResetTrap(pVCpu);
11233
11234 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11235 if ( rc == VINF_SUCCESS
11236 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11237 || rc == VERR_PAGE_NOT_PRESENT)
11238 {
11239 /* Successfully synced our nested page tables. */
11240 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11241 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11242 | HM_CHANGED_GUEST_RSP
11243 | HM_CHANGED_GUEST_RFLAGS);
11244 return VINF_SUCCESS;
11245 }
11246
11247 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11248 return rc;
11249}
11250
11251/** @} */
11252
11253/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11254/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11255/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11256
11257/** @name VM-exit exception handlers.
11258 * @{
11259 */
11260
11261/**
11262 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11263 */
11264static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11265{
11266 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11267 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11268
11269 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11270 AssertRCReturn(rc, rc);
11271
11272 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11273 {
11274 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11275 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11276
11277 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11278 * provides VM-exit instruction length. If this causes problem later,
11279 * disassemble the instruction like it's done on AMD-V. */
11280 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11281 AssertRCReturn(rc2, rc2);
11282 return rc;
11283 }
11284
11285 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11286 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11287 return rc;
11288}
11289
11290
11291/**
11292 * VM-exit exception handler for #BP (Breakpoint exception).
11293 */
11294static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11295{
11296 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11297 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11298
11299 /** @todo Try optimize this by not saving the entire guest state unless
11300 * really needed. */
11301 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11302 AssertRCReturn(rc, rc);
11303
11304 PVM pVM = pVCpu->CTX_SUFF(pVM);
11305 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11306 if (rc == VINF_EM_RAW_GUEST_TRAP)
11307 {
11308 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11309 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11310 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11311 AssertRCReturn(rc, rc);
11312
11313 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11314 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11315 }
11316
11317 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11318 return rc;
11319}
11320
11321
11322/**
11323 * VM-exit exception handler for #DB (Debug exception).
11324 */
11325static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11326{
11327 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11328 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11329 Log6(("XcptDB\n"));
11330
11331 /*
11332 * Get the DR6-like values from the exit qualification and pass it to DBGF
11333 * for processing.
11334 */
11335 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11336 AssertRCReturn(rc, rc);
11337
11338 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11339 uint64_t uDR6 = X86_DR6_INIT_VAL;
11340 uDR6 |= ( pVmxTransient->uExitQualification
11341 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11342
11343 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11344 if (rc == VINF_EM_RAW_GUEST_TRAP)
11345 {
11346 /*
11347 * The exception was for the guest. Update DR6, DR7.GD and
11348 * IA32_DEBUGCTL.LBR before forwarding it.
11349 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11350 */
11351 VMMRZCallRing3Disable(pVCpu);
11352 HM_DISABLE_PREEMPT_IF_NEEDED();
11353
11354 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11355 pMixedCtx->dr[6] |= uDR6;
11356 if (CPUMIsGuestDebugStateActive(pVCpu))
11357 ASMSetDR6(pMixedCtx->dr[6]);
11358
11359 HM_RESTORE_PREEMPT_IF_NEEDED();
11360 VMMRZCallRing3Enable(pVCpu);
11361
11362 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11363 AssertRCReturn(rc, rc);
11364
11365 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11366 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11367
11368 /* Paranoia. */
11369 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11370 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11371
11372 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11373 AssertRCReturn(rc, rc);
11374
11375 /*
11376 * Raise #DB in the guest.
11377 *
11378 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11379 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11380 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11381 *
11382 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11383 */
11384 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11385 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11386 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11387 AssertRCReturn(rc, rc);
11388 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11389 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11390 return VINF_SUCCESS;
11391 }
11392
11393 /*
11394 * Not a guest trap, must be a hypervisor related debug event then.
11395 * Update DR6 in case someone is interested in it.
11396 */
11397 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11398 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11399 CPUMSetHyperDR6(pVCpu, uDR6);
11400
11401 return rc;
11402}
11403
11404
11405/**
11406 * VM-exit exception handler for #NM (Device-not-available exception: floating
11407 * point exception).
11408 */
11409static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11410{
11411 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11412
11413 /* We require CR0 and EFER. EFER is always up-to-date. */
11414 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11415 AssertRCReturn(rc, rc);
11416
11417 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11418 VMMRZCallRing3Disable(pVCpu);
11419 HM_DISABLE_PREEMPT_IF_NEEDED();
11420
11421 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11422 if (pVmxTransient->fWasGuestFPUStateActive)
11423 {
11424 rc = VINF_EM_RAW_GUEST_TRAP;
11425 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11426 }
11427 else
11428 {
11429#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11430 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11431#endif
11432 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11433 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11434 }
11435
11436 HM_RESTORE_PREEMPT_IF_NEEDED();
11437 VMMRZCallRing3Enable(pVCpu);
11438
11439 if (rc == VINF_SUCCESS)
11440 {
11441 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11442 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11443 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11444 pVCpu->hm.s.fUseGuestFpu = true;
11445 }
11446 else
11447 {
11448 /* Forward #NM to the guest. */
11449 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11450 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11451 AssertRCReturn(rc, rc);
11452 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11453 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11454 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11455 }
11456
11457 return VINF_SUCCESS;
11458}
11459
11460
11461/**
11462 * VM-exit exception handler for #GP (General-protection exception).
11463 *
11464 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11465 */
11466static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11467{
11468 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11469 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11470
11471 int rc = VERR_INTERNAL_ERROR_5;
11472 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11473 {
11474#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11475 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11476 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11477 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11478 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11479 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11480 AssertRCReturn(rc, rc);
11481 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11482 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11483 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11484 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11485 return rc;
11486#else
11487 /* We don't intercept #GP. */
11488 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11489 NOREF(pVmxTransient);
11490 return VERR_VMX_UNEXPECTED_EXCEPTION;
11491#endif
11492 }
11493
11494 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11495 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11496
11497 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11498 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11499 AssertRCReturn(rc, rc);
11500
11501 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11502 uint32_t cbOp = 0;
11503 PVM pVM = pVCpu->CTX_SUFF(pVM);
11504 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11505 if (RT_SUCCESS(rc))
11506 {
11507 rc = VINF_SUCCESS;
11508 Assert(cbOp == pDis->cbInstr);
11509 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11510 switch (pDis->pCurInstr->uOpcode)
11511 {
11512 case OP_CLI:
11513 {
11514 pMixedCtx->eflags.Bits.u1IF = 0;
11515 pMixedCtx->rip += pDis->cbInstr;
11516 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11517 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11518 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11519 break;
11520 }
11521
11522 case OP_STI:
11523 {
11524 pMixedCtx->eflags.Bits.u1IF = 1;
11525 pMixedCtx->rip += pDis->cbInstr;
11526 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11527 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11528 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11529 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11530 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11531 break;
11532 }
11533
11534 case OP_HLT:
11535 {
11536 rc = VINF_EM_HALT;
11537 pMixedCtx->rip += pDis->cbInstr;
11538 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11539 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11540 break;
11541 }
11542
11543 case OP_POPF:
11544 {
11545 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11546 uint32_t cbParm;
11547 uint32_t uMask;
11548 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11549 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11550 {
11551 cbParm = 4;
11552 uMask = 0xffffffff;
11553 }
11554 else
11555 {
11556 cbParm = 2;
11557 uMask = 0xffff;
11558 }
11559
11560 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11561 RTGCPTR GCPtrStack = 0;
11562 X86EFLAGS Eflags;
11563 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11564 &GCPtrStack);
11565 if (RT_SUCCESS(rc))
11566 {
11567 Assert(sizeof(Eflags.u32) >= cbParm);
11568 Eflags.u32 = 0;
11569 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11570 }
11571 if (RT_FAILURE(rc))
11572 {
11573 rc = VERR_EM_INTERPRETER;
11574 break;
11575 }
11576 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11577 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11578 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11579 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11580 pMixedCtx->esp += cbParm;
11581 pMixedCtx->esp &= uMask;
11582 pMixedCtx->rip += pDis->cbInstr;
11583 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11584 | HM_CHANGED_GUEST_RSP
11585 | HM_CHANGED_GUEST_RFLAGS);
11586 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11587 if (fStepping)
11588 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11589
11590 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11591 break;
11592 }
11593
11594 case OP_PUSHF:
11595 {
11596 uint32_t cbParm;
11597 uint32_t uMask;
11598 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11599 {
11600 cbParm = 4;
11601 uMask = 0xffffffff;
11602 }
11603 else
11604 {
11605 cbParm = 2;
11606 uMask = 0xffff;
11607 }
11608
11609 /* Get the stack pointer & push the contents of eflags onto the stack. */
11610 RTGCPTR GCPtrStack = 0;
11611 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11612 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11613 if (RT_FAILURE(rc))
11614 {
11615 rc = VERR_EM_INTERPRETER;
11616 break;
11617 }
11618 X86EFLAGS Eflags = pMixedCtx->eflags;
11619 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11620 Eflags.Bits.u1RF = 0;
11621 Eflags.Bits.u1VM = 0;
11622
11623 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11624 if (RT_FAILURE(rc))
11625 {
11626 rc = VERR_EM_INTERPRETER;
11627 break;
11628 }
11629 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11630 pMixedCtx->esp -= cbParm;
11631 pMixedCtx->esp &= uMask;
11632 pMixedCtx->rip += pDis->cbInstr;
11633 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11634 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11635 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11636 break;
11637 }
11638
11639 case OP_IRET:
11640 {
11641 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11642 * instruction reference. */
11643 RTGCPTR GCPtrStack = 0;
11644 uint32_t uMask = 0xffff;
11645 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11646 uint16_t aIretFrame[3];
11647 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11648 {
11649 rc = VERR_EM_INTERPRETER;
11650 break;
11651 }
11652 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11653 &GCPtrStack);
11654 if (RT_SUCCESS(rc))
11655 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11656 if (RT_FAILURE(rc))
11657 {
11658 rc = VERR_EM_INTERPRETER;
11659 break;
11660 }
11661 pMixedCtx->eip = 0;
11662 pMixedCtx->ip = aIretFrame[0];
11663 pMixedCtx->cs.Sel = aIretFrame[1];
11664 pMixedCtx->cs.ValidSel = aIretFrame[1];
11665 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11666 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11667 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11668 pMixedCtx->sp += sizeof(aIretFrame);
11669 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11670 | HM_CHANGED_GUEST_SEGMENT_REGS
11671 | HM_CHANGED_GUEST_RSP
11672 | HM_CHANGED_GUEST_RFLAGS);
11673 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11674 if (fStepping)
11675 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11676 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11677 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11678 break;
11679 }
11680
11681 case OP_INT:
11682 {
11683 uint16_t uVector = pDis->Param1.uValue & 0xff;
11684 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11685 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11686 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11687 break;
11688 }
11689
11690 case OP_INTO:
11691 {
11692 if (pMixedCtx->eflags.Bits.u1OF)
11693 {
11694 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11695 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11696 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11697 }
11698 break;
11699 }
11700
11701 default:
11702 {
11703 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11704 EMCODETYPE_SUPERVISOR);
11705 rc = VBOXSTRICTRC_VAL(rc2);
11706 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11707 /** @todo We have to set pending-debug exceptions here when the guest is
11708 * single-stepping depending on the instruction that was interpreted. */
11709 Log4(("#GP rc=%Rrc\n", rc));
11710 break;
11711 }
11712 }
11713 }
11714 else
11715 rc = VERR_EM_INTERPRETER;
11716
11717 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11718 ("#GP Unexpected rc=%Rrc\n", rc));
11719 return rc;
11720}
11721
11722
11723#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11724/**
11725 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11726 * the exception reported in the VMX transient structure back into the VM.
11727 *
11728 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11729 * up-to-date.
11730 */
11731static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11732{
11733 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11734
11735 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11736 hmR0VmxCheckExitDueToEventDelivery(). */
11737 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11738 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11739 AssertRCReturn(rc, rc);
11740 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11741
11742#ifdef DEBUG_ramshankar
11743 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11744 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11745 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11746#endif
11747
11748 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11749 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11750 return VINF_SUCCESS;
11751}
11752#endif
11753
11754
11755/**
11756 * VM-exit exception handler for #PF (Page-fault exception).
11757 */
11758static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11759{
11760 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11761 PVM pVM = pVCpu->CTX_SUFF(pVM);
11762 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11763 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11764 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11765 AssertRCReturn(rc, rc);
11766
11767#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11768 if (pVM->hm.s.fNestedPaging)
11769 {
11770 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11771 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11772 {
11773 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11774 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11775 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11776 }
11777 else
11778 {
11779 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11780 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11781 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11782 }
11783 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11784 return rc;
11785 }
11786#else
11787 Assert(!pVM->hm.s.fNestedPaging);
11788 NOREF(pVM);
11789#endif
11790
11791 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11792 AssertRCReturn(rc, rc);
11793
11794 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11795 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11796
11797 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11798 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11799 (RTGCPTR)pVmxTransient->uExitQualification);
11800
11801 Log4(("#PF: rc=%Rrc\n", rc));
11802 if (rc == VINF_SUCCESS)
11803 {
11804 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11805 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11806 * memory? We don't update the whole state here... */
11807 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11808 | HM_CHANGED_GUEST_RSP
11809 | HM_CHANGED_GUEST_RFLAGS
11810 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11811 TRPMResetTrap(pVCpu);
11812 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11813 return rc;
11814 }
11815 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11816 {
11817 if (!pVmxTransient->fVectoringPF)
11818 {
11819 /* It's a guest page fault and needs to be reflected to the guest. */
11820 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11821 TRPMResetTrap(pVCpu);
11822 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11823 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11824 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11825 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11826 }
11827 else
11828 {
11829 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11830 TRPMResetTrap(pVCpu);
11831 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11832 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11833 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11834 }
11835
11836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11837 return VINF_SUCCESS;
11838 }
11839
11840 TRPMResetTrap(pVCpu);
11841 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11842 return rc;
11843}
11844
11845/** @} */
11846
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette