VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 52172

Last change on this file since 52172 was 52172, checked in by vboxsync, 10 years ago

VMM/HMVMXR0: Clear inhibit-interrupts force-flag when only NMI blocking is in effect. Don't start VM with NMI-window exiting control set.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 499.0 KB
Line 
1/* $Id: HMVMXR0.cpp 52172 2014-07-24 16:11:50Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
41# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_CHECK_GUEST_STATE
44# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
45# define HMVMX_ALWAYS_TRAP_PF
46# define HMVMX_ALWAYS_SWAP_FPU_STATE
47# define HMVMX_ALWAYS_FLUSH_TLB
48# define HMVMX_ALWAYS_SWAP_EFER
49#endif
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define HMVMX_IS_64BIT_HOST_MODE() (true)
57typedef RTHCUINTREG HMVMXHCUINTREG;
58#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
59extern "C" uint32_t g_fVMXIs64bitHost;
60# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
61typedef uint64_t HMVMXHCUINTREG;
62#else
63# define HMVMX_IS_64BIT_HOST_MODE() (false)
64typedef RTHCUINTREG HMVMXHCUINTREG;
65#endif
66
67/** Use the function table. */
68#define HMVMX_USE_FUNCTION_TABLE
69
70/** Determine which tagged-TLB flush handler to use. */
71#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
72#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
73#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
74#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
75
76/** @name Updated-guest-state flags.
77 * @{ */
78#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
79#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
80#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
81#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
82#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
83#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
84#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
85#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
86#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
87#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
88#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
89#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
90#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
91#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
92#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
93#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
94#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
95#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
96#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
97#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
98#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
99 | HMVMX_UPDATED_GUEST_RSP \
100 | HMVMX_UPDATED_GUEST_RFLAGS \
101 | HMVMX_UPDATED_GUEST_CR0 \
102 | HMVMX_UPDATED_GUEST_CR3 \
103 | HMVMX_UPDATED_GUEST_CR4 \
104 | HMVMX_UPDATED_GUEST_GDTR \
105 | HMVMX_UPDATED_GUEST_IDTR \
106 | HMVMX_UPDATED_GUEST_LDTR \
107 | HMVMX_UPDATED_GUEST_TR \
108 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
109 | HMVMX_UPDATED_GUEST_DEBUG \
110 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
112 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
113 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
114 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
115 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
116 | HMVMX_UPDATED_GUEST_INTR_STATE \
117 | HMVMX_UPDATED_GUEST_APIC_STATE)
118/** @} */
119
120/** @name
121 * Flags to skip redundant reads of some common VMCS fields that are not part of
122 * the guest-CPU state but are in the transient structure.
123 */
124#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
125#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
131/** @} */
132
133/** @name
134 * States of the VMCS.
135 *
136 * This does not reflect all possible VMCS states but currently only those
137 * needed for maintaining the VMCS consistently even when thread-context hooks
138 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
139 */
140#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
141#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
142#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
143/** @} */
144
145/**
146 * Exception bitmap mask for real-mode guests (real-on-v86).
147 *
148 * We need to intercept all exceptions manually (except #PF). #NM is also
149 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
150 * even in real-mode if we have Nested Paging support.
151 */
152#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
153 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
154 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
155 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
156 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
157 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
158 | RT_BIT(X86_XCPT_XF))
159
160/**
161 * Exception bitmap mask for all contributory exceptions.
162 *
163 * Page fault is deliberately excluded here as it's conditional as to whether
164 * it's contributory or benign. Page faults are handled separately.
165 */
166#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
167 | RT_BIT(X86_XCPT_DE))
168
169/** Maximum VM-instruction error number. */
170#define HMVMX_INSTR_ERROR_MAX 28
171
172/** Profiling macro. */
173#ifdef HM_PROFILE_EXIT_DISPATCH
174# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
176#else
177# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
178# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
179#endif
180
181/** Assert that preemption is disabled or covered by thread-context hooks. */
182#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
184
185/** Assert that we haven't migrated CPUs when thread-context hooks are not
186 * used. */
187#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
188 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
189 ("Illegal migration! Entered on CPU %u Current %u\n", \
190 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
191
192/** Helper macro for VM-exit handlers called unexpectedly. */
193#define HMVMX_RETURN_UNEXPECTED_EXIT() \
194 do { \
195 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
196 return VERR_VMX_UNEXPECTED_EXIT; \
197 } while (0)
198
199
200/*******************************************************************************
201* Structures and Typedefs *
202*******************************************************************************/
203/**
204 * VMX transient state.
205 *
206 * A state structure for holding miscellaneous information across
207 * VMX non-root operation and restored after the transition.
208 */
209typedef struct VMXTRANSIENT
210{
211 /** The host's rflags/eflags. */
212 RTCCUINTREG uEflags;
213#if HC_ARCH_BITS == 32
214 uint32_t u32Alignment0;
215#endif
216 /** The guest's TPR value used for TPR shadowing. */
217 uint8_t u8GuestTpr;
218 /** Alignment. */
219 uint8_t abAlignment0[7];
220
221 /** The basic VM-exit reason. */
222 uint16_t uExitReason;
223 /** Alignment. */
224 uint16_t u16Alignment0;
225 /** The VM-exit interruption error code. */
226 uint32_t uExitIntErrorCode;
227 /** The VM-exit exit qualification. */
228 uint64_t uExitQualification;
229
230 /** The VM-exit interruption-information field. */
231 uint32_t uExitIntInfo;
232 /** The VM-exit instruction-length field. */
233 uint32_t cbInstr;
234 /** The VM-exit instruction-information field. */
235 union
236 {
237 /** Plain unsigned int representation. */
238 uint32_t u;
239 /** INS and OUTS information. */
240 struct
241 {
242 uint32_t u6Reserved0 : 7;
243 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
244 uint32_t u3AddrSize : 3;
245 uint32_t u5Reserved1 : 5;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 uint32_t uReserved2 : 14;
249 } StrIo;
250 } ExitInstrInfo;
251 /** Whether the VM-entry failed or not. */
252 bool fVMEntryFailed;
253 /** Alignment. */
254 uint8_t abAlignment1[3];
255
256 /** The VM-entry interruption-information field. */
257 uint32_t uEntryIntInfo;
258 /** The VM-entry exception error code field. */
259 uint32_t uEntryXcptErrorCode;
260 /** The VM-entry instruction length field. */
261 uint32_t cbEntryInstr;
262
263 /** IDT-vectoring information field. */
264 uint32_t uIdtVectoringInfo;
265 /** IDT-vectoring error code. */
266 uint32_t uIdtVectoringErrorCode;
267
268 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
269 uint32_t fVmcsFieldsRead;
270
271 /** Whether the guest FPU was active at the time of VM-exit. */
272 bool fWasGuestFPUStateActive;
273 /** Whether the guest debug state was active at the time of VM-exit. */
274 bool fWasGuestDebugStateActive;
275 /** Whether the hyper debug state was active at the time of VM-exit. */
276 bool fWasHyperDebugStateActive;
277 /** Whether TSC-offsetting should be setup before VM-entry. */
278 bool fUpdateTscOffsettingAndPreemptTimer;
279 /** Whether the VM-exit was caused by a page-fault during delivery of a
280 * contributory exception or a page-fault. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns VBox status code.
323 * @param pVCpu Pointer to the VMCPU.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337
338/*******************************************************************************
339* Internal Functions *
340*******************************************************************************/
341static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
342static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
343static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
344 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
345#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
346static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
347#endif
348#ifndef HMVMX_USE_FUNCTION_TABLE
349DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
350# define HMVMX_EXIT_DECL static int
351#else
352# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
353#endif
354
355/** @name VM-exit handlers.
356 * @{
357 */
358static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
359static FNVMXEXITHANDLER hmR0VmxExitExtInt;
360static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
361static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
362static FNVMXEXITHANDLER hmR0VmxExitSipi;
363static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
364static FNVMXEXITHANDLER hmR0VmxExitSmi;
365static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
366static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
367static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
368static FNVMXEXITHANDLER hmR0VmxExitCpuid;
369static FNVMXEXITHANDLER hmR0VmxExitGetsec;
370static FNVMXEXITHANDLER hmR0VmxExitHlt;
371static FNVMXEXITHANDLER hmR0VmxExitInvd;
372static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
373static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
374static FNVMXEXITHANDLER hmR0VmxExitVmcall;
375static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
376static FNVMXEXITHANDLER hmR0VmxExitRsm;
377static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
378static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
379static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
380static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
381static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
382static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
383static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
384static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
385static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
386static FNVMXEXITHANDLER hmR0VmxExitMwait;
387static FNVMXEXITHANDLER hmR0VmxExitMtf;
388static FNVMXEXITHANDLER hmR0VmxExitMonitor;
389static FNVMXEXITHANDLER hmR0VmxExitPause;
390static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
391static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
392static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
393static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
394static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
395static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
396static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
397static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
398static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
399static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
400static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
401static FNVMXEXITHANDLER hmR0VmxExitRdrand;
402static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
403/** @} */
404
405static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
412static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413#endif
414static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
415
416/*******************************************************************************
417* Global Variables *
418*******************************************************************************/
419#ifdef HMVMX_USE_FUNCTION_TABLE
420
421/**
422 * VMX_EXIT dispatch table.
423 */
424static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
425{
426 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
427 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
428 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
429 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
430 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
431 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
432 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
433 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
434 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
435 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
436 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
437 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
438 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
439 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
440 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
441 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
442 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
443 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
444 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
445 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
446 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
447 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
448 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
449 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
450 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
451 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
452 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
453 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
454 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
455 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
456 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
457 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
458 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
459 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
460 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
461 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
462 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
463 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
464 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
465 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
466 /* 40 UNDEFINED */ hmR0VmxExitPause,
467 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
468 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
469 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
470 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
471 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
472 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
473 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
474 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
475 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
476 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
477 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
478 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
479 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
480 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
481 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
482 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
483 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
484 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
485 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
486};
487#endif /* HMVMX_USE_FUNCTION_TABLE */
488
489#ifdef VBOX_STRICT
490static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
491{
492 /* 0 */ "(Not Used)",
493 /* 1 */ "VMCALL executed in VMX root operation.",
494 /* 2 */ "VMCLEAR with invalid physical address.",
495 /* 3 */ "VMCLEAR with VMXON pointer.",
496 /* 4 */ "VMLAUNCH with non-clear VMCS.",
497 /* 5 */ "VMRESUME with non-launched VMCS.",
498 /* 6 */ "VMRESUME after VMXOFF",
499 /* 7 */ "VM entry with invalid control fields.",
500 /* 8 */ "VM entry with invalid host state fields.",
501 /* 9 */ "VMPTRLD with invalid physical address.",
502 /* 10 */ "VMPTRLD with VMXON pointer.",
503 /* 11 */ "VMPTRLD with incorrect revision identifier.",
504 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
505 /* 13 */ "VMWRITE to read-only VMCS component.",
506 /* 14 */ "(Not Used)",
507 /* 15 */ "VMXON executed in VMX root operation.",
508 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
509 /* 17 */ "VM entry with non-launched executing VMCS.",
510 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
511 /* 19 */ "VMCALL with non-clear VMCS.",
512 /* 20 */ "VMCALL with invalid VM-exit control fields.",
513 /* 21 */ "(Not Used)",
514 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
515 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
516 /* 24 */ "VMCALL with invalid SMM-monitor features.",
517 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
518 /* 26 */ "VM entry with events blocked by MOV SS.",
519 /* 27 */ "(Not Used)",
520 /* 28 */ "Invalid operand to INVEPT/INVVPID."
521};
522#endif /* VBOX_STRICT */
523
524
525
526/**
527 * Updates the VM's last error record. If there was a VMX instruction error,
528 * reads the error data from the VMCS and updates VCPU's last error record as
529 * well.
530 *
531 * @param pVM Pointer to the VM.
532 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
533 * VERR_VMX_UNABLE_TO_START_VM or
534 * VERR_VMX_INVALID_VMCS_FIELD).
535 * @param rc The error code.
536 */
537static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
538{
539 AssertPtr(pVM);
540 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
541 || rc == VERR_VMX_UNABLE_TO_START_VM)
542 {
543 AssertPtrReturnVoid(pVCpu);
544 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
545 }
546 pVM->hm.s.lLastError = rc;
547}
548
549
550/**
551 * Reads the VM-entry interruption-information field from the VMCS into the VMX
552 * transient structure.
553 *
554 * @returns VBox status code.
555 * @param pVmxTransient Pointer to the VMX transient structure.
556 *
557 * @remarks No-long-jump zone!!!
558 */
559DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
560{
561 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
562 AssertRCReturn(rc, rc);
563 return VINF_SUCCESS;
564}
565
566
567/**
568 * Reads the VM-entry exception error code field from the VMCS into
569 * the VMX transient structure.
570 *
571 * @returns VBox status code.
572 * @param pVmxTransient Pointer to the VMX transient structure.
573 *
574 * @remarks No-long-jump zone!!!
575 */
576DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
577{
578 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
579 AssertRCReturn(rc, rc);
580 return VINF_SUCCESS;
581}
582
583
584/**
585 * Reads the VM-entry exception error code field from the VMCS into
586 * the VMX transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600
601/**
602 * Reads the VM-exit interruption-information field from the VMCS into the VMX
603 * transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 */
608DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
611 {
612 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
613 AssertRCReturn(rc, rc);
614 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
615 }
616 return VINF_SUCCESS;
617}
618
619
620/**
621 * Reads the VM-exit interruption error code from the VMCS into the VMX
622 * transient structure.
623 *
624 * @returns VBox status code.
625 * @param pVmxTransient Pointer to the VMX transient structure.
626 */
627DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
630 {
631 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
632 AssertRCReturn(rc, rc);
633 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
634 }
635 return VINF_SUCCESS;
636}
637
638
639/**
640 * Reads the VM-exit instruction length field from the VMCS into the VMX
641 * transient structure.
642 *
643 * @returns VBox status code.
644 * @param pVCpu Pointer to the VMCPU.
645 * @param pVmxTransient Pointer to the VMX transient structure.
646 */
647DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
648{
649 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
650 {
651 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
652 AssertRCReturn(rc, rc);
653 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
654 }
655 return VINF_SUCCESS;
656}
657
658
659/**
660 * Reads the VM-exit instruction-information field from the VMCS into
661 * the VMX transient structure.
662 *
663 * @returns VBox status code.
664 * @param pVmxTransient Pointer to the VMX transient structure.
665 */
666DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
667{
668 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
669 {
670 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
671 AssertRCReturn(rc, rc);
672 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
673 }
674 return VINF_SUCCESS;
675}
676
677
678/**
679 * Reads the exit qualification from the VMCS into the VMX transient structure.
680 *
681 * @returns VBox status code.
682 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
683 * case).
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
689 {
690 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the IDT-vectoring information field from the VMCS into the VMX
700 * transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 *
705 * @remarks No-long-jump zone!!!
706 */
707DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
710 {
711 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the IDT-vectoring error code from the VMCS into the VMX
721 * transient structure.
722 *
723 * @returns VBox status code.
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 */
726DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
727{
728 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
729 {
730 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
731 AssertRCReturn(rc, rc);
732 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
733 }
734 return VINF_SUCCESS;
735}
736
737
738/**
739 * Enters VMX root mode operation on the current CPU.
740 *
741 * @returns VBox status code.
742 * @param pVM Pointer to the VM (optional, can be NULL, after
743 * a resume).
744 * @param HCPhysCpuPage Physical address of the VMXON region.
745 * @param pvCpuPage Pointer to the VMXON region.
746 */
747static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
748{
749 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
750 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
751 Assert(pvCpuPage);
752 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
753
754 if (pVM)
755 {
756 /* Write the VMCS revision dword to the VMXON region. */
757 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
758 }
759
760 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
761 RTCCUINTREG uEflags = ASMIntDisableFlags();
762
763 /* Enable the VMX bit in CR4 if necessary. */
764 RTCCUINTREG uCr4 = ASMGetCR4();
765 if (!(uCr4 & X86_CR4_VMXE))
766 ASMSetCR4(uCr4 | X86_CR4_VMXE);
767
768 /* Enter VMX root mode. */
769 int rc = VMXEnable(HCPhysCpuPage);
770 if (RT_FAILURE(rc))
771 ASMSetCR4(uCr4);
772
773 /* Restore interrupts. */
774 ASMSetFlags(uEflags);
775 return rc;
776}
777
778
779/**
780 * Exits VMX root mode operation on the current CPU.
781 *
782 * @returns VBox status code.
783 */
784static int hmR0VmxLeaveRootMode(void)
785{
786 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
787
788 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
789 RTCCUINTREG uEflags = ASMIntDisableFlags();
790
791 /* If we're for some reason not in VMX root mode, then don't leave it. */
792 RTCCUINTREG uHostCR4 = ASMGetCR4();
793
794 int rc;
795 if (uHostCR4 & X86_CR4_VMXE)
796 {
797 /* Exit VMX root mode and clear the VMX bit in CR4. */
798 VMXDisable();
799 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
800 rc = VINF_SUCCESS;
801 }
802 else
803 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
804
805 /* Restore interrupts. */
806 ASMSetFlags(uEflags);
807 return rc;
808}
809
810
811/**
812 * Allocates and maps one physically contiguous page. The allocated page is
813 * zero'd out. (Used by various VT-x structures).
814 *
815 * @returns IPRT status code.
816 * @param pMemObj Pointer to the ring-0 memory object.
817 * @param ppVirt Where to store the virtual address of the
818 * allocation.
819 * @param pPhys Where to store the physical address of the
820 * allocation.
821 */
822DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
823{
824 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
825 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
826 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
827
828 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
829 if (RT_FAILURE(rc))
830 return rc;
831 *ppVirt = RTR0MemObjAddress(*pMemObj);
832 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
833 ASMMemZero32(*ppVirt, PAGE_SIZE);
834 return VINF_SUCCESS;
835}
836
837
838/**
839 * Frees and unmaps an allocated physical page.
840 *
841 * @param pMemObj Pointer to the ring-0 memory object.
842 * @param ppVirt Where to re-initialize the virtual address of
843 * allocation as 0.
844 * @param pHCPhys Where to re-initialize the physical address of the
845 * allocation as 0.
846 */
847DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
848{
849 AssertPtr(pMemObj);
850 AssertPtr(ppVirt);
851 AssertPtr(pHCPhys);
852 if (*pMemObj != NIL_RTR0MEMOBJ)
853 {
854 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
855 AssertRC(rc);
856 *pMemObj = NIL_RTR0MEMOBJ;
857 *ppVirt = 0;
858 *pHCPhys = 0;
859 }
860}
861
862
863/**
864 * Worker function to free VT-x related structures.
865 *
866 * @returns IPRT status code.
867 * @param pVM Pointer to the VM.
868 */
869static void hmR0VmxStructsFree(PVM pVM)
870{
871 for (VMCPUID i = 0; i < pVM->cCpus; i++)
872 {
873 PVMCPU pVCpu = &pVM->aCpus[i];
874 AssertPtr(pVCpu);
875
876 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
877 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
878
879 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
880 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
881
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
883 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
884 }
885
886 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
887#ifdef VBOX_WITH_CRASHDUMP_MAGIC
888 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
889#endif
890}
891
892
893/**
894 * Worker function to allocate VT-x related VM structures.
895 *
896 * @returns IPRT status code.
897 * @param pVM Pointer to the VM.
898 */
899static int hmR0VmxStructsAlloc(PVM pVM)
900{
901 /*
902 * Initialize members up-front so we can cleanup properly on allocation failure.
903 */
904#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
905 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907 pVM->hm.s.vmx.HCPhys##a_Name = 0;
908
909#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
910 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
911 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
912 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
913
914#ifdef VBOX_WITH_CRASHDUMP_MAGIC
915 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
916#endif
917 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
918
919 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
920 for (VMCPUID i = 0; i < pVM->cCpus; i++)
921 {
922 PVMCPU pVCpu = &pVM->aCpus[i];
923 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
924 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
925 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
926 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
927 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
928 }
929#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
930#undef VMXLOCAL_INIT_VM_MEMOBJ
931
932 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
933 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
934 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
935 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
936
937 /*
938 * Allocate all the VT-x structures.
939 */
940 int rc = VINF_SUCCESS;
941#ifdef VBOX_WITH_CRASHDUMP_MAGIC
942 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
943 if (RT_FAILURE(rc))
944 goto cleanup;
945 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
946 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
947#endif
948
949 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
950 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
951 {
952 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
953 &pVM->hm.s.vmx.HCPhysApicAccess);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956 }
957
958 /*
959 * Initialize per-VCPU VT-x structures.
960 */
961 for (VMCPUID i = 0; i < pVM->cCpus; i++)
962 {
963 PVMCPU pVCpu = &pVM->aCpus[i];
964 AssertPtr(pVCpu);
965
966 /* Allocate the VM control structure (VMCS). */
967 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
968 if (RT_FAILURE(rc))
969 goto cleanup;
970
971 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
972 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
973 {
974 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
975 &pVCpu->hm.s.vmx.HCPhysVirtApic);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978 }
979
980 /*
981 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
982 * transparent accesses of specific MSRs.
983 *
984 * If the condition for enabling MSR bitmaps changes here, don't forget to
985 * update HMIsMsrBitmapsAvailable().
986 */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
988 {
989 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
990 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
994 }
995
996 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
997 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000
1001 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1002 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1003 if (RT_FAILURE(rc))
1004 goto cleanup;
1005 }
1006
1007 return VINF_SUCCESS;
1008
1009cleanup:
1010 hmR0VmxStructsFree(pVM);
1011 return rc;
1012}
1013
1014
1015/**
1016 * Does global VT-x initialization (called during module initialization).
1017 *
1018 * @returns VBox status code.
1019 */
1020VMMR0DECL(int) VMXR0GlobalInit(void)
1021{
1022#ifdef HMVMX_USE_FUNCTION_TABLE
1023 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1024# ifdef VBOX_STRICT
1025 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1026 Assert(g_apfnVMExitHandlers[i]);
1027# endif
1028#endif
1029 return VINF_SUCCESS;
1030}
1031
1032
1033/**
1034 * Does global VT-x termination (called during module termination).
1035 */
1036VMMR0DECL(void) VMXR0GlobalTerm()
1037{
1038 /* Nothing to do currently. */
1039}
1040
1041
1042/**
1043 * Sets up and activates VT-x on the current CPU.
1044 *
1045 * @returns VBox status code.
1046 * @param pCpu Pointer to the global CPU info struct.
1047 * @param pVM Pointer to the VM (can be NULL after a host resume
1048 * operation).
1049 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1050 * fEnabledByHost is true).
1051 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1052 * @a fEnabledByHost is true).
1053 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1054 * enable VT-x on the host.
1055 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1056 */
1057VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1058 void *pvMsrs)
1059{
1060 Assert(pCpu);
1061 Assert(pvMsrs);
1062 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1063
1064 /* Enable VT-x if it's not already enabled by the host. */
1065 if (!fEnabledByHost)
1066 {
1067 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1068 if (RT_FAILURE(rc))
1069 return rc;
1070 }
1071
1072 /*
1073 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1074 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1075 */
1076 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1077 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1078 {
1079 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1080 pCpu->fFlushAsidBeforeUse = false;
1081 }
1082 else
1083 pCpu->fFlushAsidBeforeUse = true;
1084
1085 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1086 ++pCpu->cTlbFlushes;
1087
1088 return VINF_SUCCESS;
1089}
1090
1091
1092/**
1093 * Deactivates VT-x on the current CPU.
1094 *
1095 * @returns VBox status code.
1096 * @param pCpu Pointer to the global CPU info struct.
1097 * @param pvCpuPage Pointer to the VMXON region.
1098 * @param HCPhysCpuPage Physical address of the VMXON region.
1099 *
1100 * @remarks This function should never be called when SUPR0EnableVTx() or
1101 * similar was used to enable VT-x on the host.
1102 */
1103VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1104{
1105 NOREF(pCpu);
1106 NOREF(pvCpuPage);
1107 NOREF(HCPhysCpuPage);
1108
1109 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1110 return hmR0VmxLeaveRootMode();
1111}
1112
1113
1114/**
1115 * Sets the permission bits for the specified MSR in the MSR bitmap.
1116 *
1117 * @param pVCpu Pointer to the VMCPU.
1118 * @param uMSR The MSR value.
1119 * @param enmRead Whether reading this MSR causes a VM-exit.
1120 * @param enmWrite Whether writing this MSR causes a VM-exit.
1121 */
1122static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1123{
1124 int32_t iBit;
1125 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1126
1127 /*
1128 * Layout:
1129 * 0x000 - 0x3ff - Low MSR read bits
1130 * 0x400 - 0x7ff - High MSR read bits
1131 * 0x800 - 0xbff - Low MSR write bits
1132 * 0xc00 - 0xfff - High MSR write bits
1133 */
1134 if (uMsr <= 0x00001FFF)
1135 iBit = uMsr;
1136 else if ( uMsr >= 0xC0000000
1137 && uMsr <= 0xC0001FFF)
1138 {
1139 iBit = (uMsr - 0xC0000000);
1140 pbMsrBitmap += 0x400;
1141 }
1142 else
1143 {
1144 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1145 return;
1146 }
1147
1148 Assert(iBit <= 0x1fff);
1149 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1150 ASMBitSet(pbMsrBitmap, iBit);
1151 else
1152 ASMBitClear(pbMsrBitmap, iBit);
1153
1154 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1155 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1156 else
1157 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1158}
1159
1160
1161#ifdef VBOX_STRICT
1162/**
1163 * Gets the permission bits for the specified MSR in the MSR bitmap.
1164 *
1165 * @returns VBox status code.
1166 * @retval VINF_SUCCESS if the specified MSR is found.
1167 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1168 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1169 *
1170 * @param pVCpu Pointer to the VMCPU.
1171 * @param uMsr The MSR.
1172 * @param penmRead Where to store the read permissions.
1173 * @param penmWrite Where to store the write permissions.
1174 */
1175static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1176{
1177 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1178 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1179 int32_t iBit;
1180 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1181
1182 /* See hmR0VmxSetMsrPermission() for the layout. */
1183 if (uMsr <= 0x00001FFF)
1184 iBit = uMsr;
1185 else if ( uMsr >= 0xC0000000
1186 && uMsr <= 0xC0001FFF)
1187 {
1188 iBit = (uMsr - 0xC0000000);
1189 pbMsrBitmap += 0x400;
1190 }
1191 else
1192 {
1193 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1194 return VERR_NOT_SUPPORTED;
1195 }
1196
1197 Assert(iBit <= 0x1fff);
1198 if (ASMBitTest(pbMsrBitmap, iBit))
1199 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1200 else
1201 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1202
1203 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1204 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1205 else
1206 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1207 return VINF_SUCCESS;
1208}
1209#endif /* VBOX_STRICT */
1210
1211
1212/**
1213 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1214 * area.
1215 *
1216 * @returns VBox status code.
1217 * @param pVCpu Pointer to the VMCPU.
1218 * @param cMsrs The number of MSRs.
1219 */
1220DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1221{
1222 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1223 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1224 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1225 {
1226 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1227 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1228 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1229 }
1230
1231 /* Update number of guest MSRs to load/store across the world-switch. */
1232 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1233 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1234
1235 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1236 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237
1238 /* Update the VCPU's copy of the MSR count. */
1239 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1240
1241 return VINF_SUCCESS;
1242}
1243
1244
1245/**
1246 * Adds a new (or updates the value of an existing) guest/host MSR
1247 * pair to be swapped during the world-switch as part of the
1248 * auto-load/store MSR area in the VMCS.
1249 *
1250 * @returns true if the MSR was added -and- its value was updated, false
1251 * otherwise.
1252 * @param pVCpu Pointer to the VMCPU.
1253 * @param uMsr The MSR.
1254 * @param uGuestMsr Value of the guest MSR.
1255 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1256 * necessary.
1257 */
1258static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1259{
1260 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1262 uint32_t i;
1263 for (i = 0; i < cMsrs; i++)
1264 {
1265 if (pGuestMsr->u32Msr == uMsr)
1266 break;
1267 pGuestMsr++;
1268 }
1269
1270 bool fAdded = false;
1271 if (i == cMsrs)
1272 {
1273 ++cMsrs;
1274 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1275 AssertRC(rc);
1276
1277 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1278 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1279 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1280
1281 fAdded = true;
1282 }
1283
1284 /* Update the MSR values in the auto-load/store MSR area. */
1285 pGuestMsr->u32Msr = uMsr;
1286 pGuestMsr->u64Value = uGuestMsrValue;
1287
1288 /* Create/update the MSR slot in the host MSR area. */
1289 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1290 pHostMsr += i;
1291 pHostMsr->u32Msr = uMsr;
1292
1293 /*
1294 * Update the host MSR only when requested by the caller AND when we're
1295 * adding it to the auto-load/store area. Otherwise, it would have been
1296 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1297 */
1298 bool fUpdatedMsrValue = false;
1299 if ( fAdded
1300 && fUpdateHostMsr)
1301 {
1302 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1303 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1304 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1305 fUpdatedMsrValue = true;
1306 }
1307
1308 return fUpdatedMsrValue;
1309}
1310
1311
1312/**
1313 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1314 * auto-load/store MSR area in the VMCS.
1315 *
1316 * @returns VBox status code.
1317 * @param pVCpu Pointer to the VMCPU.
1318 * @param uMsr The MSR.
1319 */
1320static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1321{
1322 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1323 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1324 for (uint32_t i = 0; i < cMsrs; i++)
1325 {
1326 /* Find the MSR. */
1327 if (pGuestMsr->u32Msr == uMsr)
1328 {
1329 /* If it's the last MSR, simply reduce the count. */
1330 if (i == cMsrs - 1)
1331 {
1332 --cMsrs;
1333 break;
1334 }
1335
1336 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1337 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1338 pLastGuestMsr += cMsrs - 1;
1339 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1340 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1341
1342 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1343 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1344 pLastHostMsr += cMsrs - 1;
1345 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1346 pHostMsr->u64Value = pLastHostMsr->u64Value;
1347 --cMsrs;
1348 break;
1349 }
1350 pGuestMsr++;
1351 }
1352
1353 /* Update the VMCS if the count changed (meaning the MSR was found). */
1354 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1355 {
1356 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1357 AssertRCReturn(rc, rc);
1358
1359 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1360 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1361 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1362
1363 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1364 return VINF_SUCCESS;
1365 }
1366
1367 return VERR_NOT_FOUND;
1368}
1369
1370
1371/**
1372 * Checks if the specified guest MSR is part of the auto-load/store area in
1373 * the VMCS.
1374 *
1375 * @returns true if found, false otherwise.
1376 * @param pVCpu Pointer to the VMCPU.
1377 * @param uMsr The MSR to find.
1378 */
1379static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1380{
1381 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1382 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1383
1384 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1385 {
1386 if (pGuestMsr->u32Msr == uMsr)
1387 return true;
1388 }
1389 return false;
1390}
1391
1392
1393/**
1394 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1395 *
1396 * @param pVCpu Pointer to the VMCPU.
1397 *
1398 * @remarks No-long-jump zone!!!
1399 */
1400static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1401{
1402 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1403 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1405 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1406
1407 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1408 {
1409 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1410
1411 /*
1412 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1413 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1414 */
1415 if (pHostMsr->u32Msr == MSR_K6_EFER)
1416 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1417 else
1418 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1419 }
1420
1421 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1422}
1423
1424
1425#if HC_ARCH_BITS == 64
1426/**
1427 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1428 * perform lazy restoration of the host MSRs while leaving VT-x.
1429 *
1430 * @param pVCpu Pointer to the VMCPU.
1431 *
1432 * @remarks No-long-jump zone!!!
1433 */
1434static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1435{
1436 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1437
1438 /*
1439 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1440 */
1441 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1442 {
1443 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1444 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1445 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1446 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1447 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1448 }
1449}
1450
1451
1452/**
1453 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1454 * lazily while leaving VT-x.
1455 *
1456 * @returns true if it does, false otherwise.
1457 * @param pVCpu Pointer to the VMCPU.
1458 * @param uMsr The MSR to check.
1459 */
1460static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1461{
1462 NOREF(pVCpu);
1463 switch (uMsr)
1464 {
1465 case MSR_K8_LSTAR:
1466 case MSR_K6_STAR:
1467 case MSR_K8_SF_MASK:
1468 case MSR_K8_KERNEL_GS_BASE:
1469 return true;
1470 }
1471 return false;
1472}
1473
1474
1475/**
1476 * Saves a set of guests MSRs back into the guest-CPU context.
1477 *
1478 * @param pVCpu Pointer to the VMCPU.
1479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1480 * out-of-sync. Make sure to update the required fields
1481 * before using them.
1482 *
1483 * @remarks No-long-jump zone!!!
1484 */
1485static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1486{
1487 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1488 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1489
1490 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1491 {
1492 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1493 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1494 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1495 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1496 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1497 }
1498}
1499
1500
1501/**
1502 * Loads a set of guests MSRs to allow read/passthru to the guest.
1503 *
1504 * The name of this function is slightly confusing. This function does NOT
1505 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1506 * common prefix for functions dealing with "lazy restoration" of the shared
1507 * MSRs.
1508 *
1509 * @param pVCpu Pointer to the VMCPU.
1510 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1511 * out-of-sync. Make sure to update the required fields
1512 * before using them.
1513 *
1514 * @remarks No-long-jump zone!!!
1515 */
1516static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1517{
1518 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1519 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1520
1521 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1522 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1523 {
1524#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1525 do { \
1526 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1527 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1528 else \
1529 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1530 } while (0)
1531
1532 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1533 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1534 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1535 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1536#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1537 }
1538 else
1539 {
1540 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1541 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1542 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1543 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1544 }
1545 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1546}
1547
1548
1549/**
1550 * Performs lazy restoration of the set of host MSRs if they were previously
1551 * loaded with guest MSR values.
1552 *
1553 * @param pVCpu Pointer to the VMCPU.
1554 *
1555 * @remarks No-long-jump zone!!!
1556 * @remarks The guest MSRs should have been saved back into the guest-CPU
1557 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1558 */
1559static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1560{
1561 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1562 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1563
1564 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1565 {
1566 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1567 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1568 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1569 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1570 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1571 }
1572 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1573}
1574#endif /* HC_ARCH_BITS == 64 */
1575
1576
1577/**
1578 * Verifies that our cached values of the VMCS controls are all
1579 * consistent with what's actually present in the VMCS.
1580 *
1581 * @returns VBox status code.
1582 * @param pVCpu Pointer to the VMCPU.
1583 */
1584static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1585{
1586 uint32_t u32Val;
1587 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1588 AssertRCReturn(rc, rc);
1589 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1590 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1591
1592 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1593 AssertRCReturn(rc, rc);
1594 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1595 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1596
1597 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1598 AssertRCReturn(rc, rc);
1599 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1600 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1601
1602 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1603 AssertRCReturn(rc, rc);
1604 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1605 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1606
1607 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1608 AssertRCReturn(rc, rc);
1609 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1610 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1611
1612 return VINF_SUCCESS;
1613}
1614
1615
1616#ifdef VBOX_STRICT
1617/**
1618 * Verifies that our cached host EFER value has not changed
1619 * since we cached it.
1620 *
1621 * @param pVCpu Pointer to the VMCPU.
1622 */
1623static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1624{
1625 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1626
1627 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1628 {
1629 uint64_t u64Val;
1630 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1631 AssertRC(rc);
1632
1633 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1634 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1635 }
1636}
1637
1638
1639/**
1640 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1641 * VMCS are correct.
1642 *
1643 * @param pVCpu Pointer to the VMCPU.
1644 */
1645static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1646{
1647 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1648
1649 /* Verify MSR counts in the VMCS are what we think it should be. */
1650 uint32_t cMsrs;
1651 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1652 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1653
1654 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1655 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1656
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1658 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1659
1660 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1661 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1662 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1663 {
1664 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1665 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1666 pGuestMsr->u32Msr, cMsrs));
1667
1668 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1669 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1670 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1671
1672 /* Verify that the permissions are as expected in the MSR bitmap. */
1673 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1674 {
1675 VMXMSREXITREAD enmRead;
1676 VMXMSREXITWRITE enmWrite;
1677 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1678 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1679 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1680 {
1681 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1682 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1683 }
1684 else
1685 {
1686 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1687 pGuestMsr->u32Msr, cMsrs));
1688 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1689 pGuestMsr->u32Msr, cMsrs));
1690 }
1691 }
1692 }
1693}
1694#endif /* VBOX_STRICT */
1695
1696
1697/**
1698 * Flushes the TLB using EPT.
1699 *
1700 * @returns VBox status code.
1701 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1702 * enmFlush).
1703 * @param enmFlush Type of flush.
1704 *
1705 * @remarks Caller is responsible for making sure this function is called only
1706 * when NestedPaging is supported and providing @a enmFlush that is
1707 * supported by the CPU.
1708 * @remarks Can be called with interrupts disabled.
1709 */
1710static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1711{
1712 uint64_t au64Descriptor[2];
1713 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1714 au64Descriptor[0] = 0;
1715 else
1716 {
1717 Assert(pVCpu);
1718 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1719 }
1720 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1721
1722 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1723 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1724 rc));
1725 if ( RT_SUCCESS(rc)
1726 && pVCpu)
1727 {
1728 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1729 }
1730}
1731
1732
1733/**
1734 * Flushes the TLB using VPID.
1735 *
1736 * @returns VBox status code.
1737 * @param pVM Pointer to the VM.
1738 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1739 * enmFlush).
1740 * @param enmFlush Type of flush.
1741 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1742 * on @a enmFlush).
1743 *
1744 * @remarks Can be called with interrupts disabled.
1745 */
1746static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1747{
1748 NOREF(pVM);
1749 AssertPtr(pVM);
1750 Assert(pVM->hm.s.vmx.fVpid);
1751
1752 uint64_t au64Descriptor[2];
1753 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1754 {
1755 au64Descriptor[0] = 0;
1756 au64Descriptor[1] = 0;
1757 }
1758 else
1759 {
1760 AssertPtr(pVCpu);
1761 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1762 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1763 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1764 au64Descriptor[1] = GCPtr;
1765 }
1766
1767 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1768 AssertMsg(rc == VINF_SUCCESS,
1769 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1770 if ( RT_SUCCESS(rc)
1771 && pVCpu)
1772 {
1773 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1774 }
1775}
1776
1777
1778/**
1779 * Invalidates a guest page by guest virtual address. Only relevant for
1780 * EPT/VPID, otherwise there is nothing really to invalidate.
1781 *
1782 * @returns VBox status code.
1783 * @param pVM Pointer to the VM.
1784 * @param pVCpu Pointer to the VMCPU.
1785 * @param GCVirt Guest virtual address of the page to invalidate.
1786 */
1787VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1788{
1789 AssertPtr(pVM);
1790 AssertPtr(pVCpu);
1791 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1792
1793 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1794 if (!fFlushPending)
1795 {
1796 /*
1797 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1798 * See @bugref{6043} and @bugref{6177}.
1799 *
1800 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1801 * function maybe called in a loop with individual addresses.
1802 */
1803 if (pVM->hm.s.vmx.fVpid)
1804 {
1805 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1806 {
1807 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1808 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1809 }
1810 else
1811 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1812 }
1813 else if (pVM->hm.s.fNestedPaging)
1814 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1815 }
1816
1817 return VINF_SUCCESS;
1818}
1819
1820
1821/**
1822 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1823 * otherwise there is nothing really to invalidate.
1824 *
1825 * @returns VBox status code.
1826 * @param pVM Pointer to the VM.
1827 * @param pVCpu Pointer to the VMCPU.
1828 * @param GCPhys Guest physical address of the page to invalidate.
1829 */
1830VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1831{
1832 NOREF(pVM); NOREF(GCPhys);
1833 LogFlowFunc(("%RGp\n", GCPhys));
1834
1835 /*
1836 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1837 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1838 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1839 */
1840 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1841 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1842 return VINF_SUCCESS;
1843}
1844
1845
1846/**
1847 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1848 * case where neither EPT nor VPID is supported by the CPU.
1849 *
1850 * @param pVM Pointer to the VM.
1851 * @param pVCpu Pointer to the VMCPU.
1852 * @param pCpu Pointer to the global HM struct.
1853 *
1854 * @remarks Called with interrupts disabled.
1855 */
1856static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1857{
1858 AssertPtr(pVCpu);
1859 AssertPtr(pCpu);
1860 NOREF(pVM);
1861
1862 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1863
1864 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1865#if 0
1866 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1867 pVCpu->hm.s.TlbShootdown.cPages = 0;
1868#endif
1869
1870 Assert(pCpu->idCpu != NIL_RTCPUID);
1871 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1872 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1873 pVCpu->hm.s.fForceTLBFlush = false;
1874 return;
1875}
1876
1877
1878/**
1879 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1880 *
1881 * @param pVM Pointer to the VM.
1882 * @param pVCpu Pointer to the VMCPU.
1883 * @param pCpu Pointer to the global HM CPU struct.
1884 * @remarks All references to "ASID" in this function pertains to "VPID" in
1885 * Intel's nomenclature. The reason is, to avoid confusion in compare
1886 * statements since the host-CPU copies are named "ASID".
1887 *
1888 * @remarks Called with interrupts disabled.
1889 */
1890static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1891{
1892#ifdef VBOX_WITH_STATISTICS
1893 bool fTlbFlushed = false;
1894# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1895# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1896 if (!fTlbFlushed) \
1897 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1898 } while (0)
1899#else
1900# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1901# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1902#endif
1903
1904 AssertPtr(pVM);
1905 AssertPtr(pCpu);
1906 AssertPtr(pVCpu);
1907 Assert(pCpu->idCpu != NIL_RTCPUID);
1908
1909 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1910 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1911 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1912
1913 /*
1914 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1915 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1916 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1917 */
1918 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1919 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1920 {
1921 ++pCpu->uCurrentAsid;
1922 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1923 {
1924 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1925 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1926 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1927 }
1928
1929 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1930 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1931 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1932
1933 /*
1934 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1935 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1936 */
1937 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1938 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1939 HMVMX_SET_TAGGED_TLB_FLUSHED();
1940 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1941 }
1942
1943 /* Check for explicit TLB shootdowns. */
1944 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1945 {
1946 /*
1947 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1948 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1949 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1950 * but not guest-physical mappings.
1951 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1952 */
1953 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1955 HMVMX_SET_TAGGED_TLB_FLUSHED();
1956 }
1957
1958 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1959 * where it is commented out. Support individual entry flushing
1960 * someday. */
1961#if 0
1962 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1963 {
1964 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965
1966 /*
1967 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1968 * as supported by the CPU.
1969 */
1970 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1971 {
1972 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1973 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1974 }
1975 else
1976 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1977
1978 HMVMX_SET_TAGGED_TLB_FLUSHED();
1979 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1980 pVCpu->hm.s.TlbShootdown.cPages = 0;
1981 }
1982#endif
1983
1984 pVCpu->hm.s.fForceTLBFlush = false;
1985
1986 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1987
1988 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1989 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1990 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1991 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1992 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1993 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1994 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1995 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1996 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1997
1998 /* Update VMCS with the VPID. */
1999 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2000 AssertRC(rc);
2001
2002#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2003}
2004
2005
2006/**
2007 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2008 *
2009 * @returns VBox status code.
2010 * @param pVM Pointer to the VM.
2011 * @param pVCpu Pointer to the VMCPU.
2012 * @param pCpu Pointer to the global HM CPU struct.
2013 *
2014 * @remarks Called with interrupts disabled.
2015 */
2016static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2017{
2018 AssertPtr(pVM);
2019 AssertPtr(pVCpu);
2020 AssertPtr(pCpu);
2021 Assert(pCpu->idCpu != NIL_RTCPUID);
2022 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2023 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2024
2025 /*
2026 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2027 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2028 */
2029 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2030 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2031 {
2032 pVCpu->hm.s.fForceTLBFlush = true;
2033 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034 }
2035
2036 /* Check for explicit TLB shootdown flushes. */
2037 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2038 {
2039 pVCpu->hm.s.fForceTLBFlush = true;
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2041 }
2042
2043 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2044 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2045
2046 if (pVCpu->hm.s.fForceTLBFlush)
2047 {
2048 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2049 pVCpu->hm.s.fForceTLBFlush = false;
2050 }
2051 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2052 * where it is commented out. Support individual entry flushing
2053 * someday. */
2054#if 0
2055 else
2056 {
2057 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2058 {
2059 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2060 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2061 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2062 }
2063 else
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2065
2066 pVCpu->hm.s.TlbShootdown.cPages = 0;
2067 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2068 }
2069#endif
2070}
2071
2072
2073/**
2074 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2075 *
2076 * @returns VBox status code.
2077 * @param pVM Pointer to the VM.
2078 * @param pVCpu Pointer to the VMCPU.
2079 * @param pCpu Pointer to the global HM CPU struct.
2080 *
2081 * @remarks Called with interrupts disabled.
2082 */
2083static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2084{
2085 AssertPtr(pVM);
2086 AssertPtr(pVCpu);
2087 AssertPtr(pCpu);
2088 Assert(pCpu->idCpu != NIL_RTCPUID);
2089 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2090 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2091
2092 /*
2093 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2094 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2095 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2096 */
2097 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2098 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2099 {
2100 pVCpu->hm.s.fForceTLBFlush = true;
2101 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2102 }
2103
2104 /* Check for explicit TLB shootdown flushes. */
2105 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2106 {
2107 /*
2108 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2109 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2110 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2111 */
2112 pVCpu->hm.s.fForceTLBFlush = true;
2113 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2114 }
2115
2116 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2117 if (pVCpu->hm.s.fForceTLBFlush)
2118 {
2119 ++pCpu->uCurrentAsid;
2120 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2121 {
2122 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2123 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2124 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2125 }
2126
2127 pVCpu->hm.s.fForceTLBFlush = false;
2128 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2129 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2130 if (pCpu->fFlushAsidBeforeUse)
2131 {
2132 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2133 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2134 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2135 {
2136 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2137 pCpu->fFlushAsidBeforeUse = false;
2138 }
2139 else
2140 {
2141 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2142 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2143 }
2144 }
2145 }
2146 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2147 * where it is commented out. Support individual entry flushing
2148 * someday. */
2149#if 0
2150 else
2151 {
2152 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2153 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2154 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2155 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2156
2157 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2158 {
2159 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2160 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2161 {
2162 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2163 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2164 }
2165 else
2166 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2167
2168 pVCpu->hm.s.TlbShootdown.cPages = 0;
2169 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2170 }
2171 else
2172 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2173 }
2174#endif
2175
2176 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2177 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2178 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2179 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2180 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2181 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2182 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2183
2184 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2185 AssertRC(rc);
2186}
2187
2188
2189/**
2190 * Flushes the guest TLB entry based on CPU capabilities.
2191 *
2192 * @param pVCpu Pointer to the VMCPU.
2193 * @param pCpu Pointer to the global HM CPU struct.
2194 */
2195DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2196{
2197#ifdef HMVMX_ALWAYS_FLUSH_TLB
2198 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2199#endif
2200 PVM pVM = pVCpu->CTX_SUFF(pVM);
2201 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2202 {
2203 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2204 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2205 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2206 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2207 default:
2208 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2209 break;
2210 }
2211
2212 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2213 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2214
2215 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2216}
2217
2218
2219/**
2220 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2221 * TLB entries from the host TLB before VM-entry.
2222 *
2223 * @returns VBox status code.
2224 * @param pVM Pointer to the VM.
2225 */
2226static int hmR0VmxSetupTaggedTlb(PVM pVM)
2227{
2228 /*
2229 * Determine optimal flush type for Nested Paging.
2230 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2231 * guest execution (see hmR3InitFinalizeR0()).
2232 */
2233 if (pVM->hm.s.fNestedPaging)
2234 {
2235 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2236 {
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2238 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2239 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2240 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2241 else
2242 {
2243 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2244 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2245 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2246 }
2247
2248 /* Make sure the write-back cacheable memory type for EPT is supported. */
2249 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2250 {
2251 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2252 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2253 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2254 }
2255 }
2256 else
2257 {
2258 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2259 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2261 }
2262 }
2263
2264 /*
2265 * Determine optimal flush type for VPID.
2266 */
2267 if (pVM->hm.s.vmx.fVpid)
2268 {
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2270 {
2271 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2272 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2273 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2274 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2275 else
2276 {
2277 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2278 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2279 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2280 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2281 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2282 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2283 pVM->hm.s.vmx.fVpid = false;
2284 }
2285 }
2286 else
2287 {
2288 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2289 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2290 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2291 pVM->hm.s.vmx.fVpid = false;
2292 }
2293 }
2294
2295 /*
2296 * Setup the handler for flushing tagged-TLBs.
2297 */
2298 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2299 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2300 else if (pVM->hm.s.fNestedPaging)
2301 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2302 else if (pVM->hm.s.vmx.fVpid)
2303 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2304 else
2305 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2306 return VINF_SUCCESS;
2307}
2308
2309
2310/**
2311 * Sets up pin-based VM-execution controls in the VMCS.
2312 *
2313 * @returns VBox status code.
2314 * @param pVM Pointer to the VM.
2315 * @param pVCpu Pointer to the VMCPU.
2316 */
2317static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2318{
2319 AssertPtr(pVM);
2320 AssertPtr(pVCpu);
2321
2322 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2323 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2324
2325 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2326 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2327
2328 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2329 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2330
2331 /* Enable the VMX preemption timer. */
2332 if (pVM->hm.s.vmx.fUsePreemptTimer)
2333 {
2334 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2335 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2336 }
2337
2338 if ((val & zap) != val)
2339 {
2340 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2341 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2342 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2343 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2344 }
2345
2346 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2347 AssertRCReturn(rc, rc);
2348
2349 /* Update VCPU with the currently set pin-based VM-execution controls. */
2350 pVCpu->hm.s.vmx.u32PinCtls = val;
2351 return rc;
2352}
2353
2354
2355/**
2356 * Sets up processor-based VM-execution controls in the VMCS.
2357 *
2358 * @returns VBox status code.
2359 * @param pVM Pointer to the VM.
2360 * @param pVMCPU Pointer to the VMCPU.
2361 */
2362static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2363{
2364 AssertPtr(pVM);
2365 AssertPtr(pVCpu);
2366
2367 int rc = VERR_INTERNAL_ERROR_5;
2368 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2378
2379 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2380 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2381 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2382 {
2383 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2384 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2385 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2386 }
2387
2388 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2389 if (!pVM->hm.s.fNestedPaging)
2390 {
2391 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2392 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2393 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2394 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2395 }
2396
2397 /* Use TPR shadowing if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2399 {
2400 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2401 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2402 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2403 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2404 AssertRCReturn(rc, rc);
2405
2406 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2407 /* CR8 writes cause a VM-exit based on TPR threshold. */
2408 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2409 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2410 }
2411 else
2412 {
2413 /*
2414 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2415 * Set this control only for 64-bit guests.
2416 */
2417 if (pVM->hm.s.fAllow64BitGuests)
2418 {
2419 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2421 }
2422 }
2423
2424 /* Use MSR-bitmaps if supported by the CPU. */
2425 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2426 {
2427 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2428
2429 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2431 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2432 AssertRCReturn(rc, rc);
2433
2434 /*
2435 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2436 * automatically using dedicated fields in the VMCS.
2437 */
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443
2444#if HC_ARCH_BITS == 64
2445 /*
2446 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2447 */
2448 if (pVM->hm.s.fAllow64BitGuests)
2449 {
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454 }
2455#endif
2456 }
2457
2458 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2459 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2460 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2461
2462 if ((val & zap) != val)
2463 {
2464 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2465 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2466 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2467 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2468 }
2469
2470 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2471 AssertRCReturn(rc, rc);
2472
2473 /* Update VCPU with the currently set processor-based VM-execution controls. */
2474 pVCpu->hm.s.vmx.u32ProcCtls = val;
2475
2476 /*
2477 * Secondary processor-based VM-execution controls.
2478 */
2479 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2480 {
2481 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2482 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2483
2484 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2485 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2486
2487 if (pVM->hm.s.fNestedPaging)
2488 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2489 else
2490 {
2491 /*
2492 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2493 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2494 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2495 */
2496 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2497 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2498 }
2499
2500 if (pVM->hm.s.vmx.fVpid)
2501 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2502
2503 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2505
2506 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2507 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2508 * done dynamically. */
2509 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2510 {
2511 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2512 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2513 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2514 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2515 AssertRCReturn(rc, rc);
2516 }
2517
2518 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2519 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2520
2521 if ((val & zap) != val)
2522 {
2523 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2524 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2525 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2526 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2527 }
2528
2529 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2530 AssertRCReturn(rc, rc);
2531
2532 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2533 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2534 }
2535 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2536 {
2537 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2538 "available\n"));
2539 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2540 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2541 }
2542
2543 return VINF_SUCCESS;
2544}
2545
2546
2547/**
2548 * Sets up miscellaneous (everything other than Pin & Processor-based
2549 * VM-execution) control fields in the VMCS.
2550 *
2551 * @returns VBox status code.
2552 * @param pVM Pointer to the VM.
2553 * @param pVCpu Pointer to the VMCPU.
2554 */
2555static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2556{
2557 NOREF(pVM);
2558 AssertPtr(pVM);
2559 AssertPtr(pVCpu);
2560
2561 int rc = VERR_GENERAL_FAILURE;
2562
2563 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2564#if 0
2565 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2566 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2567 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2568
2569 /*
2570 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2571 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2572 * We thus use the exception bitmap to control it rather than use both.
2573 */
2574 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2575 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2576
2577 /** @todo Explore possibility of using IO-bitmaps. */
2578 /* All IO & IOIO instructions cause VM-exits. */
2579 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2580 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2581
2582 /* Initialize the MSR-bitmap area. */
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2584 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2585 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2586#endif
2587
2588 /* Setup MSR auto-load/store area. */
2589 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2590 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2591 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2592 AssertRCReturn(rc, rc);
2593 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2594 AssertRCReturn(rc, rc);
2595
2596 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2597 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2598 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2599 AssertRCReturn(rc, rc);
2600
2601 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2602 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2603 AssertRCReturn(rc, rc);
2604
2605 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2606#if 0
2607 /* Setup debug controls */
2608 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2609 AssertRCReturn(rc, rc);
2610 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2611 AssertRCReturn(rc, rc);
2612#endif
2613
2614 return rc;
2615}
2616
2617
2618/**
2619 * Sets up the initial exception bitmap in the VMCS based on static conditions
2620 * (i.e. conditions that cannot ever change after starting the VM).
2621 *
2622 * @returns VBox status code.
2623 * @param pVM Pointer to the VM.
2624 * @param pVCpu Pointer to the VMCPU.
2625 */
2626static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2627{
2628 AssertPtr(pVM);
2629 AssertPtr(pVCpu);
2630
2631 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2632
2633 uint32_t u32XcptBitmap = 0;
2634
2635 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2636 if (!pVM->hm.s.fNestedPaging)
2637 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2638
2639 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2640 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2641 AssertRCReturn(rc, rc);
2642 return rc;
2643}
2644
2645
2646/**
2647 * Sets up the initial guest-state mask. The guest-state mask is consulted
2648 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2649 * for the nested virtualization case (as it would cause a VM-exit).
2650 *
2651 * @param pVCpu Pointer to the VMCPU.
2652 */
2653static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2654{
2655 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2656 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2657 return VINF_SUCCESS;
2658}
2659
2660
2661/**
2662 * Does per-VM VT-x initialization.
2663 *
2664 * @returns VBox status code.
2665 * @param pVM Pointer to the VM.
2666 */
2667VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2668{
2669 LogFlowFunc(("pVM=%p\n", pVM));
2670
2671 int rc = hmR0VmxStructsAlloc(pVM);
2672 if (RT_FAILURE(rc))
2673 {
2674 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2675 return rc;
2676 }
2677
2678 return VINF_SUCCESS;
2679}
2680
2681
2682/**
2683 * Does per-VM VT-x termination.
2684 *
2685 * @returns VBox status code.
2686 * @param pVM Pointer to the VM.
2687 */
2688VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2689{
2690 LogFlowFunc(("pVM=%p\n", pVM));
2691
2692#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2693 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2694 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2695#endif
2696 hmR0VmxStructsFree(pVM);
2697 return VINF_SUCCESS;
2698}
2699
2700
2701/**
2702 * Sets up the VM for execution under VT-x.
2703 * This function is only called once per-VM during initialization.
2704 *
2705 * @returns VBox status code.
2706 * @param pVM Pointer to the VM.
2707 */
2708VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2709{
2710 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2711 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2712
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715 /*
2716 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2717 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2718 */
2719 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2720 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2721 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2722 || !pVM->hm.s.vmx.pRealModeTSS))
2723 {
2724 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2725 return VERR_INTERNAL_ERROR;
2726 }
2727
2728#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2729 /*
2730 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2731 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2732 */
2733 if ( pVM->hm.s.fAllow64BitGuests
2734 && !HMVMX_IS_64BIT_HOST_MODE())
2735 {
2736 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2737 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2738 }
2739#endif
2740
2741 /* Initialize these always, see hmR3InitFinalizeR0().*/
2742 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2743 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2744
2745 /* Setup the tagged-TLB flush handlers. */
2746 int rc = hmR0VmxSetupTaggedTlb(pVM);
2747 if (RT_FAILURE(rc))
2748 {
2749 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2750 return rc;
2751 }
2752
2753 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2754 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2755#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2756 if ( HMVMX_IS_64BIT_HOST_MODE()
2757 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2758 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2759 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2760 {
2761 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2762 }
2763#endif
2764
2765 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2766 {
2767 PVMCPU pVCpu = &pVM->aCpus[i];
2768 AssertPtr(pVCpu);
2769 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2770
2771 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2772 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2773
2774 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2775 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2776 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2777
2778 /* Set revision dword at the beginning of the VMCS structure. */
2779 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2780
2781 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2782 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2783 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2784 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2785
2786 /* Load this VMCS as the current VMCS. */
2787 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2788 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2789 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2790
2791 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2793 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2794
2795 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2801 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2802
2803 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2805 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2806
2807 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2809 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2810
2811#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2812 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815#endif
2816
2817 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2818 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2820 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2821
2822 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2823
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2825 }
2826
2827 return VINF_SUCCESS;
2828}
2829
2830
2831/**
2832 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2833 * the VMCS.
2834 *
2835 * @returns VBox status code.
2836 * @param pVM Pointer to the VM.
2837 * @param pVCpu Pointer to the VMCPU.
2838 */
2839DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2840{
2841 NOREF(pVM); NOREF(pVCpu);
2842
2843 RTCCUINTREG uReg = ASMGetCR0();
2844 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2845 AssertRCReturn(rc, rc);
2846
2847#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2848 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2849 if (HMVMX_IS_64BIT_HOST_MODE())
2850 {
2851 uint64_t uRegCR3 = HMR0Get64bitCR3();
2852 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2853 }
2854 else
2855#endif
2856 {
2857 uReg = ASMGetCR3();
2858 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2859 }
2860 AssertRCReturn(rc, rc);
2861
2862 uReg = ASMGetCR4();
2863 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2864 AssertRCReturn(rc, rc);
2865 return rc;
2866}
2867
2868
2869#if HC_ARCH_BITS == 64
2870/**
2871 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2872 * requirements. See hmR0VmxSaveHostSegmentRegs().
2873 */
2874# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2875 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2876 { \
2877 bool fValidSelector = true; \
2878 if ((selValue) & X86_SEL_LDT) \
2879 { \
2880 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2881 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2882 } \
2883 if (fValidSelector) \
2884 { \
2885 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2886 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2887 } \
2888 (selValue) = 0; \
2889 }
2890#endif
2891
2892
2893/**
2894 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2895 * the host-state area in the VMCS.
2896 *
2897 * @returns VBox status code.
2898 * @param pVM Pointer to the VM.
2899 * @param pVCpu Pointer to the VMCPU.
2900 */
2901DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2902{
2903 NOREF(pVM);
2904 int rc = VERR_INTERNAL_ERROR_5;
2905
2906#if HC_ARCH_BITS == 64
2907 /*
2908 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2909 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2910 */
2911 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2912 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2913#endif
2914
2915 /*
2916 * Host DS, ES, FS and GS segment registers.
2917 */
2918#if HC_ARCH_BITS == 64
2919 RTSEL uSelDS = ASMGetDS();
2920 RTSEL uSelES = ASMGetES();
2921 RTSEL uSelFS = ASMGetFS();
2922 RTSEL uSelGS = ASMGetGS();
2923#else
2924 RTSEL uSelDS = 0;
2925 RTSEL uSelES = 0;
2926 RTSEL uSelFS = 0;
2927 RTSEL uSelGS = 0;
2928#endif
2929
2930 /* Recalculate which host-state bits need to be manually restored. */
2931 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2932
2933 /*
2934 * Host CS and SS segment registers.
2935 */
2936#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2937 RTSEL uSelCS;
2938 RTSEL uSelSS;
2939 if (HMVMX_IS_64BIT_HOST_MODE())
2940 {
2941 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2942 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2943 }
2944 else
2945 {
2946 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2947 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2948 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2949 }
2950#else
2951 RTSEL uSelCS = ASMGetCS();
2952 RTSEL uSelSS = ASMGetSS();
2953#endif
2954
2955 /*
2956 * Host TR segment register.
2957 */
2958 RTSEL uSelTR = ASMGetTR();
2959
2960#if HC_ARCH_BITS == 64
2961 /*
2962 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2963 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2964 */
2965 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2966 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2967 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2968 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2969# undef VMXLOCAL_ADJUST_HOST_SEG
2970#endif
2971
2972 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2973 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2974 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2975 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2976 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2977 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2978 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2979 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2980 Assert(uSelCS);
2981 Assert(uSelTR);
2982
2983 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2984#if 0
2985 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2986 Assert(uSelSS != 0);
2987#endif
2988
2989 /* Write these host selector fields into the host-state area in the VMCS. */
2990 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2991 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2992#if HC_ARCH_BITS == 64
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2994 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2997#endif
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2999
3000 /*
3001 * Host GDTR and IDTR.
3002 */
3003 RTGDTR Gdtr;
3004 RT_ZERO(Gdtr);
3005#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3006 if (HMVMX_IS_64BIT_HOST_MODE())
3007 {
3008 X86XDTR64 Gdtr64;
3009 X86XDTR64 Idtr64;
3010 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3011 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3012 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3013
3014 Gdtr.cbGdt = Gdtr64.cb;
3015 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3016 }
3017 else
3018#endif
3019 {
3020 RTIDTR Idtr;
3021 ASMGetGDTR(&Gdtr);
3022 ASMGetIDTR(&Idtr);
3023 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3024 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3025
3026#if HC_ARCH_BITS == 64
3027 /*
3028 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3029 * maximum limit (0xffff) on every VM-exit.
3030 */
3031 if (Gdtr.cbGdt != 0xffff)
3032 {
3033 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3034 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3035 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3036 }
3037
3038 /*
3039 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3040 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3041 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3042 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3043 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3044 * hosts where we are pretty sure it won't cause trouble.
3045 */
3046# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3047 if (Idtr.cbIdt < 0x0fff)
3048# else
3049 if (Idtr.cbIdt != 0xffff)
3050# endif
3051 {
3052 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3053 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3054 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3055 }
3056#endif
3057 }
3058
3059 /*
3060 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3061 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3062 */
3063 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3064 {
3065 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3066 return VERR_VMX_INVALID_HOST_STATE;
3067 }
3068
3069 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3070#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3071 if (HMVMX_IS_64BIT_HOST_MODE())
3072 {
3073 /* We need the 64-bit TR base for hybrid darwin. */
3074 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3075 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3076 }
3077 else
3078#endif
3079 {
3080 uintptr_t uTRBase;
3081#if HC_ARCH_BITS == 64
3082 uTRBase = X86DESC64_BASE(pDesc);
3083
3084 /*
3085 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3086 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3087 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3088 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3089 *
3090 * [1] See Intel spec. 3.5 "System Descriptor Types".
3091 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3092 */
3093 Assert(pDesc->System.u4Type == 11);
3094 if ( pDesc->System.u16LimitLow != 0x67
3095 || pDesc->System.u4LimitHigh)
3096 {
3097 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3098 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3099
3100 /* Store the GDTR here as we need it while restoring TR. */
3101 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3102 }
3103#else
3104 uTRBase = X86DESC_BASE(pDesc);
3105#endif
3106 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3107 }
3108 AssertRCReturn(rc, rc);
3109
3110 /*
3111 * Host FS base and GS base.
3112 */
3113#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3114 if (HMVMX_IS_64BIT_HOST_MODE())
3115 {
3116 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3117 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3118 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3119 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3120
3121# if HC_ARCH_BITS == 64
3122 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3123 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3124 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3125 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3126 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3127# endif
3128 }
3129#endif
3130 return rc;
3131}
3132
3133
3134/**
3135 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3136 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3137 * the host after every successful VM-exit.
3138 *
3139 * @returns VBox status code.
3140 * @param pVM Pointer to the VM.
3141 * @param pVCpu Pointer to the VMCPU.
3142 *
3143 * @remarks No-long-jump zone!!!
3144 */
3145DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3146{
3147 NOREF(pVM);
3148
3149 AssertPtr(pVCpu);
3150 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3151
3152 int rc = VINF_SUCCESS;
3153#if HC_ARCH_BITS == 64
3154 if (pVM->hm.s.fAllow64BitGuests)
3155 hmR0VmxLazySaveHostMsrs(pVCpu);
3156#endif
3157
3158 /*
3159 * Host Sysenter MSRs.
3160 */
3161 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3162 AssertRCReturn(rc, rc);
3163#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3164 if (HMVMX_IS_64BIT_HOST_MODE())
3165 {
3166 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3167 AssertRCReturn(rc, rc);
3168 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3169 }
3170 else
3171 {
3172 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3173 AssertRCReturn(rc, rc);
3174 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3175 }
3176#elif HC_ARCH_BITS == 32
3177 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3178 AssertRCReturn(rc, rc);
3179 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3180#else
3181 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3182 AssertRCReturn(rc, rc);
3183 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3184#endif
3185 AssertRCReturn(rc, rc);
3186
3187 /*
3188 * Host EFER MSR.
3189 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3190 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3191 */
3192 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3193 {
3194 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3195 AssertRCReturn(rc, rc);
3196 }
3197
3198 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3199 * hmR0VmxLoadGuestExitCtls() !! */
3200
3201 return rc;
3202}
3203
3204
3205/**
3206 * Figures out if we need to swap the EFER MSR which is
3207 * particularly expensive.
3208 *
3209 * We check all relevant bits. For now, that's everything
3210 * besides LMA/LME, as these two bits are handled by VM-entry,
3211 * see hmR0VmxLoadGuestExitCtls() and
3212 * hmR0VMxLoadGuestEntryCtls().
3213 *
3214 * @returns true if we need to load guest EFER, false otherwise.
3215 * @param pVCpu Pointer to the VMCPU.
3216 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3217 * out-of-sync. Make sure to update the required fields
3218 * before using them.
3219 *
3220 * @remarks Requires EFER, CR4.
3221 * @remarks No-long-jump zone!!!
3222 */
3223static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3224{
3225#ifdef HMVMX_ALWAYS_SWAP_EFER
3226 return true;
3227#endif
3228
3229#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3230 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3231 if (CPUMIsGuestInLongMode(pVCpu))
3232 return false;
3233#endif
3234
3235 PVM pVM = pVCpu->CTX_SUFF(pVM);
3236 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3237 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3238
3239 /*
3240 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3241 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3242 */
3243 if ( CPUMIsGuestInLongMode(pVCpu)
3244 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3245 {
3246 return true;
3247 }
3248
3249 /*
3250 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3251 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3252 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3253 */
3254 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3255 && (pMixedCtx->cr0 & X86_CR0_PG)
3256 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3257 {
3258 /* Assert that host is PAE capable. */
3259 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3260 return true;
3261 }
3262
3263 /** @todo Check the latest Intel spec. for any other bits,
3264 * like SMEP/SMAP? */
3265 return false;
3266}
3267
3268
3269/**
3270 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3271 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3272 * controls".
3273 *
3274 * @returns VBox status code.
3275 * @param pVCpu Pointer to the VMCPU.
3276 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3277 * out-of-sync. Make sure to update the required fields
3278 * before using them.
3279 *
3280 * @remarks Requires EFER.
3281 * @remarks No-long-jump zone!!!
3282 */
3283DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3284{
3285 int rc = VINF_SUCCESS;
3286 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3287 {
3288 PVM pVM = pVCpu->CTX_SUFF(pVM);
3289 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3290 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3291
3292 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3293 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3294
3295 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3296 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3297 {
3298 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3299 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3300 }
3301 else
3302 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3303
3304 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3305 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3306 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3307 {
3308 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3309 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3310 }
3311
3312 /*
3313 * The following should -not- be set (since we're not in SMM mode):
3314 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3315 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3316 */
3317
3318 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3319 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3320
3321 if ((val & zap) != val)
3322 {
3323 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3324 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3325 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3326 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3327 }
3328
3329 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3330 AssertRCReturn(rc, rc);
3331
3332 /* Update VCPU with the currently set VM-exit controls. */
3333 pVCpu->hm.s.vmx.u32EntryCtls = val;
3334 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3335 }
3336 return rc;
3337}
3338
3339
3340/**
3341 * Sets up the VM-exit controls in the VMCS.
3342 *
3343 * @returns VBox status code.
3344 * @param pVM Pointer to the VM.
3345 * @param pVCpu Pointer to the VMCPU.
3346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3347 * out-of-sync. Make sure to update the required fields
3348 * before using them.
3349 *
3350 * @remarks Requires EFER.
3351 */
3352DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3353{
3354 NOREF(pMixedCtx);
3355
3356 int rc = VINF_SUCCESS;
3357 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3358 {
3359 PVM pVM = pVCpu->CTX_SUFF(pVM);
3360 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3361 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3362
3363 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3364 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3365
3366 /*
3367 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3368 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3369 */
3370#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3371 if (HMVMX_IS_64BIT_HOST_MODE())
3372 {
3373 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3374 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3375 }
3376 else
3377 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3378#else
3379 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3380 {
3381 /* The switcher returns to long mode, EFER is managed by the switcher. */
3382 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3383 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3384 }
3385 else
3386 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3387#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3388
3389 /* If the newer VMCS fields for managing EFER exists, use it. */
3390 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3391 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3392 {
3393 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3394 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3395 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3396 }
3397
3398 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3399 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3400
3401 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3402 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3403 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3404
3405 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3406 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3407
3408 if ((val & zap) != val)
3409 {
3410 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3411 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3412 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3413 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3414 }
3415
3416 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3417 AssertRCReturn(rc, rc);
3418
3419 /* Update VCPU with the currently set VM-exit controls. */
3420 pVCpu->hm.s.vmx.u32ExitCtls = val;
3421 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3422 }
3423 return rc;
3424}
3425
3426
3427/**
3428 * Loads the guest APIC and related state.
3429 *
3430 * @returns VBox status code.
3431 * @param pVM Pointer to the VM.
3432 * @param pVCpu Pointer to the VMCPU.
3433 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3434 * out-of-sync. Make sure to update the required fields
3435 * before using them.
3436 */
3437DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3438{
3439 NOREF(pMixedCtx);
3440
3441 int rc = VINF_SUCCESS;
3442 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3443 {
3444 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3445 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3446 {
3447 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3448
3449 bool fPendingIntr = false;
3450 uint8_t u8Tpr = 0;
3451 uint8_t u8PendingIntr = 0;
3452 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3453 AssertRCReturn(rc, rc);
3454
3455 /*
3456 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3457 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3458 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3459 * the interrupt when we VM-exit for other reasons.
3460 */
3461 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3462 uint32_t u32TprThreshold = 0;
3463 if (fPendingIntr)
3464 {
3465 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3466 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3467 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3468 if (u8PendingPriority <= u8TprPriority)
3469 u32TprThreshold = u8PendingPriority;
3470 else
3471 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3472 }
3473 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3474
3475 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3476 AssertRCReturn(rc, rc);
3477 }
3478
3479 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3480 }
3481 return rc;
3482}
3483
3484
3485/**
3486 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3487 *
3488 * @returns Guest's interruptibility-state.
3489 * @param pVCpu Pointer to the VMCPU.
3490 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3491 * out-of-sync. Make sure to update the required fields
3492 * before using them.
3493 *
3494 * @remarks No-long-jump zone!!!
3495 */
3496DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3497{
3498 /*
3499 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3500 */
3501 uint32_t uIntrState = 0;
3502 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3503 {
3504 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3505 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3506 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3507 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3508 {
3509 if (pMixedCtx->eflags.Bits.u1IF)
3510 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3511 else
3512 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3513 }
3514 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3515 }
3516
3517 /*
3518 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3519 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3520 * setting this would block host-NMIs and IRET will not clear the blocking.
3521 *
3522 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3523 */
3524 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3525 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3526 {
3527 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3528 }
3529
3530 return uIntrState;
3531}
3532
3533
3534/**
3535 * Loads the guest's interruptibility-state into the guest-state area in the
3536 * VMCS.
3537 *
3538 * @returns VBox status code.
3539 * @param pVCpu Pointer to the VMCPU.
3540 * @param uIntrState The interruptibility-state to set.
3541 */
3542static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3543{
3544 NOREF(pVCpu);
3545 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3546 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3547 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3548 AssertRCReturn(rc, rc);
3549 return rc;
3550}
3551
3552
3553/**
3554 * Loads the guest's RIP into the guest-state area in the VMCS.
3555 *
3556 * @returns VBox status code.
3557 * @param pVCpu Pointer to the VMCPU.
3558 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3559 * out-of-sync. Make sure to update the required fields
3560 * before using them.
3561 *
3562 * @remarks No-long-jump zone!!!
3563 */
3564static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3565{
3566 int rc = VINF_SUCCESS;
3567 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3568 {
3569 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3570 AssertRCReturn(rc, rc);
3571
3572 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3573 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3574 HMCPU_CF_VALUE(pVCpu)));
3575 }
3576 return rc;
3577}
3578
3579
3580/**
3581 * Loads the guest's RSP into the guest-state area in the VMCS.
3582 *
3583 * @returns VBox status code.
3584 * @param pVCpu Pointer to the VMCPU.
3585 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3586 * out-of-sync. Make sure to update the required fields
3587 * before using them.
3588 *
3589 * @remarks No-long-jump zone!!!
3590 */
3591static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3592{
3593 int rc = VINF_SUCCESS;
3594 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3595 {
3596 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3597 AssertRCReturn(rc, rc);
3598
3599 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3600 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3601 }
3602 return rc;
3603}
3604
3605
3606/**
3607 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3608 *
3609 * @returns VBox status code.
3610 * @param pVCpu Pointer to the VMCPU.
3611 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3612 * out-of-sync. Make sure to update the required fields
3613 * before using them.
3614 *
3615 * @remarks No-long-jump zone!!!
3616 */
3617static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3618{
3619 int rc = VINF_SUCCESS;
3620 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3621 {
3622 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3623 Let us assert it as such and use 32-bit VMWRITE. */
3624 Assert(!(pMixedCtx->rflags.u64 >> 32));
3625 X86EFLAGS Eflags = pMixedCtx->eflags;
3626 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3627 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3628
3629 /*
3630 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3631 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3632 */
3633 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3634 {
3635 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3636 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3637 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3638 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3639 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3640 }
3641
3642 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3643 AssertRCReturn(rc, rc);
3644
3645 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3646 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3647 }
3648 return rc;
3649}
3650
3651
3652/**
3653 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3654 *
3655 * @returns VBox status code.
3656 * @param pVCpu Pointer to the VMCPU.
3657 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3658 * out-of-sync. Make sure to update the required fields
3659 * before using them.
3660 *
3661 * @remarks No-long-jump zone!!!
3662 */
3663DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3664{
3665 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3666 AssertRCReturn(rc, rc);
3667 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3668 AssertRCReturn(rc, rc);
3669 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3670 AssertRCReturn(rc, rc);
3671 return rc;
3672}
3673
3674
3675/**
3676 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3677 * CR0 is partially shared with the host and we have to consider the FPU bits.
3678 *
3679 * @returns VBox status code.
3680 * @param pVM Pointer to the VM.
3681 * @param pVCpu Pointer to the VMCPU.
3682 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3683 * out-of-sync. Make sure to update the required fields
3684 * before using them.
3685 *
3686 * @remarks No-long-jump zone!!!
3687 */
3688static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3689{
3690 /*
3691 * Guest CR0.
3692 * Guest FPU.
3693 */
3694 int rc = VINF_SUCCESS;
3695 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3696 {
3697 Assert(!(pMixedCtx->cr0 >> 32));
3698 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3699 PVM pVM = pVCpu->CTX_SUFF(pVM);
3700
3701 /* The guest's view (read access) of its CR0 is unblemished. */
3702 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3703 AssertRCReturn(rc, rc);
3704 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3705
3706 /* Setup VT-x's view of the guest CR0. */
3707 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3708 if (pVM->hm.s.fNestedPaging)
3709 {
3710 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3711 {
3712 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3713 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3714 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3715 }
3716 else
3717 {
3718 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3719 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3720 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3721 }
3722
3723 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3724 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3725 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3726
3727 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3728 AssertRCReturn(rc, rc);
3729 }
3730 else
3731 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3732
3733 /*
3734 * Guest FPU bits.
3735 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3736 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3737 */
3738 u32GuestCR0 |= X86_CR0_NE;
3739 bool fInterceptNM = false;
3740 if (CPUMIsGuestFPUStateActive(pVCpu))
3741 {
3742 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3743 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3744 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3745 }
3746 else
3747 {
3748 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3749 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3750 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3751 }
3752
3753 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3754 bool fInterceptMF = false;
3755 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3756 fInterceptMF = true;
3757
3758 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3759 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3760 {
3761 Assert(PDMVmmDevHeapIsEnabled(pVM));
3762 Assert(pVM->hm.s.vmx.pRealModeTSS);
3763 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3764 fInterceptNM = true;
3765 fInterceptMF = true;
3766 }
3767 else
3768 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3769
3770 if (fInterceptNM)
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3772 else
3773 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3774
3775 if (fInterceptMF)
3776 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3777 else
3778 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3779
3780 /* Additional intercepts for debugging, define these yourself explicitly. */
3781#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3782 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3783 | RT_BIT(X86_XCPT_BP)
3784 | RT_BIT(X86_XCPT_DB)
3785 | RT_BIT(X86_XCPT_DE)
3786 | RT_BIT(X86_XCPT_NM)
3787 | RT_BIT(X86_XCPT_TS)
3788 | RT_BIT(X86_XCPT_UD)
3789 | RT_BIT(X86_XCPT_NP)
3790 | RT_BIT(X86_XCPT_SS)
3791 | RT_BIT(X86_XCPT_GP)
3792 | RT_BIT(X86_XCPT_PF)
3793 | RT_BIT(X86_XCPT_MF)
3794 ;
3795#elif defined(HMVMX_ALWAYS_TRAP_PF)
3796 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3797#endif
3798
3799 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3800
3801 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3802 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3803 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3804 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3805 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3806 else
3807 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3808
3809 u32GuestCR0 |= uSetCR0;
3810 u32GuestCR0 &= uZapCR0;
3811 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3812
3813 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3814 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3815 AssertRCReturn(rc, rc);
3816 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3817 AssertRCReturn(rc, rc);
3818 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3819 uZapCR0));
3820
3821 /*
3822 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3823 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3824 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3825 */
3826 uint32_t u32CR0Mask = 0;
3827 u32CR0Mask = X86_CR0_PE
3828 | X86_CR0_NE
3829 | X86_CR0_WP
3830 | X86_CR0_PG
3831 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3832 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3833 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3834
3835 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3836 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3837 * and @bugref{6944}. */
3838#if 0
3839 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3840 u32CR0Mask &= ~X86_CR0_PE;
3841#endif
3842 if (pVM->hm.s.fNestedPaging)
3843 u32CR0Mask &= ~X86_CR0_WP;
3844
3845 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3846 if (fInterceptNM)
3847 {
3848 u32CR0Mask |= X86_CR0_TS
3849 | X86_CR0_MP;
3850 }
3851
3852 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3853 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3854 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3855 AssertRCReturn(rc, rc);
3856 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3857
3858 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3859 }
3860 return rc;
3861}
3862
3863
3864/**
3865 * Loads the guest control registers (CR3, CR4) into the guest-state area
3866 * in the VMCS.
3867 *
3868 * @returns VBox status code.
3869 * @param pVM Pointer to the VM.
3870 * @param pVCpu Pointer to the VMCPU.
3871 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3872 * out-of-sync. Make sure to update the required fields
3873 * before using them.
3874 *
3875 * @remarks No-long-jump zone!!!
3876 */
3877static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3878{
3879 int rc = VINF_SUCCESS;
3880 PVM pVM = pVCpu->CTX_SUFF(pVM);
3881
3882 /*
3883 * Guest CR2.
3884 * It's always loaded in the assembler code. Nothing to do here.
3885 */
3886
3887 /*
3888 * Guest CR3.
3889 */
3890 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3891 {
3892 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3893 if (pVM->hm.s.fNestedPaging)
3894 {
3895 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3896
3897 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3898 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3899 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3900 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3901
3902 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3903 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3904 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3905
3906 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3907 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3908 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3909 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3910
3911 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3912 AssertRCReturn(rc, rc);
3913 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3914
3915 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3916 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3917 {
3918 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3919 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3920 {
3921 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3922 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3923 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3924 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3925 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3926 }
3927
3928 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3929 have Unrestricted Execution to handle the guest when it's not using paging. */
3930 GCPhysGuestCR3 = pMixedCtx->cr3;
3931 }
3932 else
3933 {
3934 /*
3935 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3936 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3937 * EPT takes care of translating it to host-physical addresses.
3938 */
3939 RTGCPHYS GCPhys;
3940 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3941 Assert(PDMVmmDevHeapIsEnabled(pVM));
3942
3943 /* We obtain it here every time as the guest could have relocated this PCI region. */
3944 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3945 AssertRCReturn(rc, rc);
3946
3947 GCPhysGuestCR3 = GCPhys;
3948 }
3949
3950 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3951 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3952 }
3953 else
3954 {
3955 /* Non-nested paging case, just use the hypervisor's CR3. */
3956 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3957
3958 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3959 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3960 }
3961 AssertRCReturn(rc, rc);
3962
3963 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3964 }
3965
3966 /*
3967 * Guest CR4.
3968 */
3969 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3970 {
3971 Assert(!(pMixedCtx->cr4 >> 32));
3972 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3973
3974 /* The guest's view of its CR4 is unblemished. */
3975 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3976 AssertRCReturn(rc, rc);
3977 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3978
3979 /* Setup VT-x's view of the guest CR4. */
3980 /*
3981 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3982 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3983 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3984 */
3985 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3986 {
3987 Assert(pVM->hm.s.vmx.pRealModeTSS);
3988 Assert(PDMVmmDevHeapIsEnabled(pVM));
3989 u32GuestCR4 &= ~X86_CR4_VME;
3990 }
3991
3992 if (pVM->hm.s.fNestedPaging)
3993 {
3994 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3995 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3996 {
3997 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3998 u32GuestCR4 |= X86_CR4_PSE;
3999 /* Our identity mapping is a 32-bit page directory. */
4000 u32GuestCR4 &= ~X86_CR4_PAE;
4001 }
4002 /* else use guest CR4.*/
4003 }
4004 else
4005 {
4006 /*
4007 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4008 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4009 */
4010 switch (pVCpu->hm.s.enmShadowMode)
4011 {
4012 case PGMMODE_REAL: /* Real-mode. */
4013 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4014 case PGMMODE_32_BIT: /* 32-bit paging. */
4015 {
4016 u32GuestCR4 &= ~X86_CR4_PAE;
4017 break;
4018 }
4019
4020 case PGMMODE_PAE: /* PAE paging. */
4021 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4022 {
4023 u32GuestCR4 |= X86_CR4_PAE;
4024 break;
4025 }
4026
4027 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4028 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4029#ifdef VBOX_ENABLE_64_BITS_GUESTS
4030 break;
4031#endif
4032 default:
4033 AssertFailed();
4034 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4035 }
4036 }
4037
4038 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4039 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4040 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4041 u32GuestCR4 |= uSetCR4;
4042 u32GuestCR4 &= uZapCR4;
4043
4044 /* Write VT-x's view of the guest CR4 into the VMCS. */
4045 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4046 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4047 AssertRCReturn(rc, rc);
4048
4049 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4050 uint32_t u32CR4Mask = 0;
4051 u32CR4Mask = X86_CR4_VME
4052 | X86_CR4_PAE
4053 | X86_CR4_PGE
4054 | X86_CR4_PSE
4055 | X86_CR4_VMXE;
4056 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4057 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4058 AssertRCReturn(rc, rc);
4059
4060 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4061 }
4062 return rc;
4063}
4064
4065
4066/**
4067 * Loads the guest debug registers into the guest-state area in the VMCS.
4068 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4069 *
4070 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4071 *
4072 * @returns VBox status code.
4073 * @param pVCpu Pointer to the VMCPU.
4074 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4075 * out-of-sync. Make sure to update the required fields
4076 * before using them.
4077 *
4078 * @remarks No-long-jump zone!!!
4079 */
4080static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4081{
4082 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4083 return VINF_SUCCESS;
4084
4085#ifdef VBOX_STRICT
4086 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4087 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4088 {
4089 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4090 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4091 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4092 }
4093#endif
4094
4095 int rc;
4096 PVM pVM = pVCpu->CTX_SUFF(pVM);
4097 bool fInterceptDB = false;
4098 bool fInterceptMovDRx = false;
4099 if ( pVCpu->hm.s.fSingleInstruction
4100 || DBGFIsStepping(pVCpu))
4101 {
4102 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4103 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4104 {
4105 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4106 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4107 AssertRCReturn(rc, rc);
4108 Assert(fInterceptDB == false);
4109 }
4110 else
4111 {
4112 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4113 pVCpu->hm.s.fClearTrapFlag = true;
4114 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4115 fInterceptDB = true;
4116 }
4117 }
4118
4119 if ( fInterceptDB
4120 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4121 {
4122 /*
4123 * Use the combined guest and host DRx values found in the hypervisor
4124 * register set because the debugger has breakpoints active or someone
4125 * is single stepping on the host side without a monitor trap flag.
4126 *
4127 * Note! DBGF expects a clean DR6 state before executing guest code.
4128 */
4129#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4130 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4131 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4132 {
4133 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4134 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4135 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4136 }
4137 else
4138#endif
4139 if (!CPUMIsHyperDebugStateActive(pVCpu))
4140 {
4141 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4142 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4143 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4144 }
4145
4146 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4147 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4148 AssertRCReturn(rc, rc);
4149
4150 pVCpu->hm.s.fUsingHyperDR7 = true;
4151 fInterceptDB = true;
4152 fInterceptMovDRx = true;
4153 }
4154 else
4155 {
4156 /*
4157 * If the guest has enabled debug registers, we need to load them prior to
4158 * executing guest code so they'll trigger at the right time.
4159 */
4160 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4161 {
4162#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4163 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4164 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4165 {
4166 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4167 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4168 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4169 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4170 }
4171 else
4172#endif
4173 if (!CPUMIsGuestDebugStateActive(pVCpu))
4174 {
4175 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4176 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4177 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4178 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4179 }
4180 Assert(!fInterceptDB);
4181 Assert(!fInterceptMovDRx);
4182 }
4183 /*
4184 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4185 * must intercept #DB in order to maintain a correct DR6 guest value.
4186 */
4187#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4188 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4189 && !CPUMIsGuestDebugStateActive(pVCpu))
4190#else
4191 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4192#endif
4193 {
4194 fInterceptMovDRx = true;
4195 fInterceptDB = true;
4196 }
4197
4198 /* Update guest DR7. */
4199 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4200 AssertRCReturn(rc, rc);
4201
4202 pVCpu->hm.s.fUsingHyperDR7 = false;
4203 }
4204
4205 /*
4206 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4207 */
4208 if (fInterceptDB)
4209 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4210 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4211 {
4212#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4213 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4214#endif
4215 }
4216 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4217 AssertRCReturn(rc, rc);
4218
4219 /*
4220 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4221 */
4222 if (fInterceptMovDRx)
4223 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4224 else
4225 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4226 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4227 AssertRCReturn(rc, rc);
4228
4229 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4230 return VINF_SUCCESS;
4231}
4232
4233
4234#ifdef VBOX_STRICT
4235/**
4236 * Strict function to validate segment registers.
4237 *
4238 * @remarks ASSUMES CR0 is up to date.
4239 */
4240static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4241{
4242 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4243 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4244 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4245 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4246 && ( !CPUMIsGuestInRealModeEx(pCtx)
4247 && !CPUMIsGuestInV86ModeEx(pCtx)))
4248 {
4249 /* Protected mode checks */
4250 /* CS */
4251 Assert(pCtx->cs.Attr.n.u1Present);
4252 Assert(!(pCtx->cs.Attr.u & 0xf00));
4253 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4254 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4255 || !(pCtx->cs.Attr.n.u1Granularity));
4256 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4257 || (pCtx->cs.Attr.n.u1Granularity));
4258 /* CS cannot be loaded with NULL in protected mode. */
4259 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4260 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4261 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4262 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4263 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4264 else
4265 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4266 /* SS */
4267 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4268 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4269 if ( !(pCtx->cr0 & X86_CR0_PE)
4270 || pCtx->cs.Attr.n.u4Type == 3)
4271 {
4272 Assert(!pCtx->ss.Attr.n.u2Dpl);
4273 }
4274 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4275 {
4276 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4277 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4278 Assert(pCtx->ss.Attr.n.u1Present);
4279 Assert(!(pCtx->ss.Attr.u & 0xf00));
4280 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4281 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4282 || !(pCtx->ss.Attr.n.u1Granularity));
4283 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4284 || (pCtx->ss.Attr.n.u1Granularity));
4285 }
4286 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4287 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4288 {
4289 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4290 Assert(pCtx->ds.Attr.n.u1Present);
4291 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4292 Assert(!(pCtx->ds.Attr.u & 0xf00));
4293 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4294 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4295 || !(pCtx->ds.Attr.n.u1Granularity));
4296 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4297 || (pCtx->ds.Attr.n.u1Granularity));
4298 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4299 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4300 }
4301 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4302 {
4303 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4304 Assert(pCtx->es.Attr.n.u1Present);
4305 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4306 Assert(!(pCtx->es.Attr.u & 0xf00));
4307 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4308 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4309 || !(pCtx->es.Attr.n.u1Granularity));
4310 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4311 || (pCtx->es.Attr.n.u1Granularity));
4312 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4313 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4314 }
4315 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4316 {
4317 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4318 Assert(pCtx->fs.Attr.n.u1Present);
4319 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4320 Assert(!(pCtx->fs.Attr.u & 0xf00));
4321 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4322 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4323 || !(pCtx->fs.Attr.n.u1Granularity));
4324 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4325 || (pCtx->fs.Attr.n.u1Granularity));
4326 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4327 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4328 }
4329 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4330 {
4331 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4332 Assert(pCtx->gs.Attr.n.u1Present);
4333 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4334 Assert(!(pCtx->gs.Attr.u & 0xf00));
4335 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4336 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4337 || !(pCtx->gs.Attr.n.u1Granularity));
4338 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4339 || (pCtx->gs.Attr.n.u1Granularity));
4340 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4341 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4342 }
4343 /* 64-bit capable CPUs. */
4344# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4345 if (HMVMX_IS_64BIT_HOST_MODE())
4346 {
4347 Assert(!(pCtx->cs.u64Base >> 32));
4348 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4349 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4350 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4351 }
4352# endif
4353 }
4354 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4355 || ( CPUMIsGuestInRealModeEx(pCtx)
4356 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4357 {
4358 /* Real and v86 mode checks. */
4359 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4360 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4361 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4362 {
4363 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4364 }
4365 else
4366 {
4367 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4368 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4369 }
4370
4371 /* CS */
4372 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4373 Assert(pCtx->cs.u32Limit == 0xffff);
4374 Assert(u32CSAttr == 0xf3);
4375 /* SS */
4376 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4377 Assert(pCtx->ss.u32Limit == 0xffff);
4378 Assert(u32SSAttr == 0xf3);
4379 /* DS */
4380 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4381 Assert(pCtx->ds.u32Limit == 0xffff);
4382 Assert(u32DSAttr == 0xf3);
4383 /* ES */
4384 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4385 Assert(pCtx->es.u32Limit == 0xffff);
4386 Assert(u32ESAttr == 0xf3);
4387 /* FS */
4388 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4389 Assert(pCtx->fs.u32Limit == 0xffff);
4390 Assert(u32FSAttr == 0xf3);
4391 /* GS */
4392 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4393 Assert(pCtx->gs.u32Limit == 0xffff);
4394 Assert(u32GSAttr == 0xf3);
4395 /* 64-bit capable CPUs. */
4396# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4397 if (HMVMX_IS_64BIT_HOST_MODE())
4398 {
4399 Assert(!(pCtx->cs.u64Base >> 32));
4400 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4401 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4402 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4403 }
4404# endif
4405 }
4406}
4407#endif /* VBOX_STRICT */
4408
4409
4410/**
4411 * Writes a guest segment register into the guest-state area in the VMCS.
4412 *
4413 * @returns VBox status code.
4414 * @param pVCpu Pointer to the VMCPU.
4415 * @param idxSel Index of the selector in the VMCS.
4416 * @param idxLimit Index of the segment limit in the VMCS.
4417 * @param idxBase Index of the segment base in the VMCS.
4418 * @param idxAccess Index of the access rights of the segment in the VMCS.
4419 * @param pSelReg Pointer to the segment selector.
4420 *
4421 * @remarks No-long-jump zone!!!
4422 */
4423static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4424 uint32_t idxAccess, PCPUMSELREG pSelReg)
4425{
4426 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4427 AssertRCReturn(rc, rc);
4428 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4429 AssertRCReturn(rc, rc);
4430 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4431 AssertRCReturn(rc, rc);
4432
4433 uint32_t u32Access = pSelReg->Attr.u;
4434 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4435 {
4436 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4437 u32Access = 0xf3;
4438 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4439 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4440 }
4441 else
4442 {
4443 /*
4444 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4445 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4446 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4447 * loaded in protected-mode have their attribute as 0.
4448 */
4449 if (!u32Access)
4450 u32Access = X86DESCATTR_UNUSABLE;
4451 }
4452
4453 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4454 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4455 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4456
4457 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4458 AssertRCReturn(rc, rc);
4459 return rc;
4460}
4461
4462
4463/**
4464 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4465 * into the guest-state area in the VMCS.
4466 *
4467 * @returns VBox status code.
4468 * @param pVM Pointer to the VM.
4469 * @param pVCPU Pointer to the VMCPU.
4470 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4471 * out-of-sync. Make sure to update the required fields
4472 * before using them.
4473 *
4474 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4475 * @remarks No-long-jump zone!!!
4476 */
4477static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4478{
4479 int rc = VERR_INTERNAL_ERROR_5;
4480 PVM pVM = pVCpu->CTX_SUFF(pVM);
4481
4482 /*
4483 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4484 */
4485 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4486 {
4487 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4488 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4489 {
4490 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4491 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4492 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4493 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4494 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4495 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4496 }
4497
4498#ifdef VBOX_WITH_REM
4499 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4500 {
4501 Assert(pVM->hm.s.vmx.pRealModeTSS);
4502 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4503 if ( pVCpu->hm.s.vmx.fWasInRealMode
4504 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4505 {
4506 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4507 in real-mode (e.g. OpenBSD 4.0) */
4508 REMFlushTBs(pVM);
4509 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4510 pVCpu->hm.s.vmx.fWasInRealMode = false;
4511 }
4512 }
4513#endif
4514 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4515 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4516 AssertRCReturn(rc, rc);
4517 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4518 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4519 AssertRCReturn(rc, rc);
4520 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4521 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4522 AssertRCReturn(rc, rc);
4523 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4524 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4525 AssertRCReturn(rc, rc);
4526 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4527 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4528 AssertRCReturn(rc, rc);
4529 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4530 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4531 AssertRCReturn(rc, rc);
4532
4533#ifdef VBOX_STRICT
4534 /* Validate. */
4535 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4536#endif
4537
4538 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4539 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4540 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4541 }
4542
4543 /*
4544 * Guest TR.
4545 */
4546 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4547 {
4548 /*
4549 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4550 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4551 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4552 */
4553 uint16_t u16Sel = 0;
4554 uint32_t u32Limit = 0;
4555 uint64_t u64Base = 0;
4556 uint32_t u32AccessRights = 0;
4557
4558 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4559 {
4560 u16Sel = pMixedCtx->tr.Sel;
4561 u32Limit = pMixedCtx->tr.u32Limit;
4562 u64Base = pMixedCtx->tr.u64Base;
4563 u32AccessRights = pMixedCtx->tr.Attr.u;
4564 }
4565 else
4566 {
4567 Assert(pVM->hm.s.vmx.pRealModeTSS);
4568 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4569
4570 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4571 RTGCPHYS GCPhys;
4572 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4573 AssertRCReturn(rc, rc);
4574
4575 X86DESCATTR DescAttr;
4576 DescAttr.u = 0;
4577 DescAttr.n.u1Present = 1;
4578 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4579
4580 u16Sel = 0;
4581 u32Limit = HM_VTX_TSS_SIZE;
4582 u64Base = GCPhys; /* in real-mode phys = virt. */
4583 u32AccessRights = DescAttr.u;
4584 }
4585
4586 /* Validate. */
4587 Assert(!(u16Sel & RT_BIT(2)));
4588 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4589 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4590 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4591 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4592 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4593 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4594 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4595 Assert( (u32Limit & 0xfff) == 0xfff
4596 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4597 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4598 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4599
4600 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4601 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4602 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4603 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4604
4605 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4606 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4607 }
4608
4609 /*
4610 * Guest GDTR.
4611 */
4612 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4613 {
4614 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4615 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4616
4617 /* Validate. */
4618 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4619
4620 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4621 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4622 }
4623
4624 /*
4625 * Guest LDTR.
4626 */
4627 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4628 {
4629 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4630 uint32_t u32Access = 0;
4631 if (!pMixedCtx->ldtr.Attr.u)
4632 u32Access = X86DESCATTR_UNUSABLE;
4633 else
4634 u32Access = pMixedCtx->ldtr.Attr.u;
4635
4636 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4637 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4638 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4639 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4640
4641 /* Validate. */
4642 if (!(u32Access & X86DESCATTR_UNUSABLE))
4643 {
4644 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4645 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4646 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4647 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4648 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4649 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4650 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4651 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4652 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4653 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4654 }
4655
4656 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4657 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4658 }
4659
4660 /*
4661 * Guest IDTR.
4662 */
4663 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4664 {
4665 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4666 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4667
4668 /* Validate. */
4669 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4670
4671 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4672 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4673 }
4674
4675 return VINF_SUCCESS;
4676}
4677
4678
4679/**
4680 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4681 * areas. These MSRs will automatically be loaded to the host CPU on every
4682 * successful VM entry and stored from the host CPU on every successful VM-exit.
4683 *
4684 * This also creates/updates MSR slots for the host MSRs. The actual host
4685 * MSR values are -not- updated here for performance reasons. See
4686 * hmR0VmxSaveHostMsrs().
4687 *
4688 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4689 *
4690 * @returns VBox status code.
4691 * @param pVCpu Pointer to the VMCPU.
4692 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4693 * out-of-sync. Make sure to update the required fields
4694 * before using them.
4695 *
4696 * @remarks No-long-jump zone!!!
4697 */
4698static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4699{
4700 AssertPtr(pVCpu);
4701 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4702
4703 /*
4704 * MSRs that we use the auto-load/store MSR area in the VMCS.
4705 */
4706 PVM pVM = pVCpu->CTX_SUFF(pVM);
4707 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4708 {
4709 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4710#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4711 if (pVM->hm.s.fAllow64BitGuests)
4712 {
4713 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4714 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4715 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4716 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4717# ifdef DEBUG
4718 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4719 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4720 {
4721 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4722 pMsr->u64Value));
4723 }
4724# endif
4725 }
4726#endif
4727 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4728 }
4729
4730 /*
4731 * Guest Sysenter MSRs.
4732 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4733 * VM-exits on WRMSRs for these MSRs.
4734 */
4735 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4736 {
4737 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4738 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4739 }
4740
4741 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4742 {
4743 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4744 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4745 }
4746
4747 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4748 {
4749 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4750 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4751 }
4752
4753 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4754 {
4755 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4756 {
4757 /*
4758 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4759 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4760 */
4761 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4762 {
4763 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4764 AssertRCReturn(rc,rc);
4765 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4766 }
4767 else
4768 {
4769 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4770 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4771 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4772 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4773 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4774 }
4775 }
4776 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4777 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4778 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4779 }
4780
4781 return VINF_SUCCESS;
4782}
4783
4784
4785/**
4786 * Loads the guest activity state into the guest-state area in the VMCS.
4787 *
4788 * @returns VBox status code.
4789 * @param pVCpu Pointer to the VMCPU.
4790 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4791 * out-of-sync. Make sure to update the required fields
4792 * before using them.
4793 *
4794 * @remarks No-long-jump zone!!!
4795 */
4796static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4797{
4798 NOREF(pCtx);
4799 /** @todo See if we can make use of other states, e.g.
4800 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4801 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4802 {
4803 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4804 AssertRCReturn(rc, rc);
4805
4806 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4807 }
4808 return VINF_SUCCESS;
4809}
4810
4811
4812/**
4813 * Sets up the appropriate function to run guest code.
4814 *
4815 * @returns VBox status code.
4816 * @param pVCpu Pointer to the VMCPU.
4817 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4818 * out-of-sync. Make sure to update the required fields
4819 * before using them.
4820 *
4821 * @remarks No-long-jump zone!!!
4822 */
4823static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4824{
4825 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4826 {
4827#ifndef VBOX_ENABLE_64_BITS_GUESTS
4828 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4829#endif
4830 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4831#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4832 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4833 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4834 {
4835 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4836 {
4837 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4838 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4839 | HM_CHANGED_VMX_EXIT_CTLS
4840 | HM_CHANGED_VMX_ENTRY_CTLS
4841 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4842 }
4843 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4844 }
4845#else
4846 /* 64-bit host or hybrid host. */
4847 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4848#endif
4849 }
4850 else
4851 {
4852 /* Guest is not in long mode, use the 32-bit handler. */
4853#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4854 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4855 {
4856 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4857 {
4858 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4859 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4860 | HM_CHANGED_VMX_EXIT_CTLS
4861 | HM_CHANGED_VMX_ENTRY_CTLS
4862 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4863 }
4864 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4865 }
4866#else
4867 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4868#endif
4869 }
4870 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4871 return VINF_SUCCESS;
4872}
4873
4874
4875/**
4876 * Wrapper for running the guest code in VT-x.
4877 *
4878 * @returns VBox strict status code.
4879 * @param pVM Pointer to the VM.
4880 * @param pVCpu Pointer to the VMCPU.
4881 * @param pCtx Pointer to the guest-CPU context.
4882 *
4883 * @remarks No-long-jump zone!!!
4884 */
4885DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4886{
4887 /*
4888 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4889 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4890 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4891 */
4892 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4893 /** @todo Add stats for resume vs launch. */
4894#ifdef VBOX_WITH_KERNEL_USING_XMM
4895 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4896#else
4897 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4898#endif
4899}
4900
4901
4902/**
4903 * Reports world-switch error and dumps some useful debug info.
4904 *
4905 * @param pVM Pointer to the VM.
4906 * @param pVCpu Pointer to the VMCPU.
4907 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4908 * @param pCtx Pointer to the guest-CPU context.
4909 * @param pVmxTransient Pointer to the VMX transient structure (only
4910 * exitReason updated).
4911 */
4912static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4913{
4914 Assert(pVM);
4915 Assert(pVCpu);
4916 Assert(pCtx);
4917 Assert(pVmxTransient);
4918 HMVMX_ASSERT_PREEMPT_SAFE();
4919
4920 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4921 switch (rcVMRun)
4922 {
4923 case VERR_VMX_INVALID_VMXON_PTR:
4924 AssertFailed();
4925 break;
4926 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4927 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4928 {
4929 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4930 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4931 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4932 AssertRC(rc);
4933
4934 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4935 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4936 Cannot do it here as we may have been long preempted. */
4937
4938#ifdef VBOX_STRICT
4939 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4940 pVmxTransient->uExitReason));
4941 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4942 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4943 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4944 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4945 else
4946 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4947 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4948 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4949
4950 /* VMX control bits. */
4951 uint32_t u32Val;
4952 uint64_t u64Val;
4953 HMVMXHCUINTREG uHCReg;
4954 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4955 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4956 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4957 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4958 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4959 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4960 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4961 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4962 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4963 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4964 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4965 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4986 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4987 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4988 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4989 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4990 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4991 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4992 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4993 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4994 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4995 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4996
4997 /* Guest bits. */
4998 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4999 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5000 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5001 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5002 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5003 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5004 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5005 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5006
5007 /* Host bits. */
5008 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5009 Log4(("Host CR0 %#RHr\n", uHCReg));
5010 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5011 Log4(("Host CR3 %#RHr\n", uHCReg));
5012 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5013 Log4(("Host CR4 %#RHr\n", uHCReg));
5014
5015 RTGDTR HostGdtr;
5016 PCX86DESCHC pDesc;
5017 ASMGetGDTR(&HostGdtr);
5018 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5019 Log4(("Host CS %#08x\n", u32Val));
5020 if (u32Val < HostGdtr.cbGdt)
5021 {
5022 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5023 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5024 }
5025
5026 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5027 Log4(("Host DS %#08x\n", u32Val));
5028 if (u32Val < HostGdtr.cbGdt)
5029 {
5030 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5031 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5032 }
5033
5034 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5035 Log4(("Host ES %#08x\n", u32Val));
5036 if (u32Val < HostGdtr.cbGdt)
5037 {
5038 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5039 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5040 }
5041
5042 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5043 Log4(("Host FS %#08x\n", u32Val));
5044 if (u32Val < HostGdtr.cbGdt)
5045 {
5046 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5047 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5048 }
5049
5050 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5051 Log4(("Host GS %#08x\n", u32Val));
5052 if (u32Val < HostGdtr.cbGdt)
5053 {
5054 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5055 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5056 }
5057
5058 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5059 Log4(("Host SS %#08x\n", u32Val));
5060 if (u32Val < HostGdtr.cbGdt)
5061 {
5062 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5063 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5064 }
5065
5066 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5067 Log4(("Host TR %#08x\n", u32Val));
5068 if (u32Val < HostGdtr.cbGdt)
5069 {
5070 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5071 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5072 }
5073
5074 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5075 Log4(("Host TR Base %#RHv\n", uHCReg));
5076 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5077 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5078 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5079 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5080 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5081 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5082 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5083 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5085 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5087 Log4(("Host RSP %#RHv\n", uHCReg));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5089 Log4(("Host RIP %#RHv\n", uHCReg));
5090# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5091 if (HMVMX_IS_64BIT_HOST_MODE())
5092 {
5093 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5094 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5095 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5096 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5097 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5098 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5099 }
5100# endif
5101#endif /* VBOX_STRICT */
5102 break;
5103 }
5104
5105 default:
5106 /* Impossible */
5107 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5108 break;
5109 }
5110 NOREF(pVM); NOREF(pCtx);
5111}
5112
5113
5114#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5115#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5116# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5117#endif
5118#ifdef VBOX_STRICT
5119static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5120{
5121 switch (idxField)
5122 {
5123 case VMX_VMCS_GUEST_RIP:
5124 case VMX_VMCS_GUEST_RSP:
5125 case VMX_VMCS_GUEST_SYSENTER_EIP:
5126 case VMX_VMCS_GUEST_SYSENTER_ESP:
5127 case VMX_VMCS_GUEST_GDTR_BASE:
5128 case VMX_VMCS_GUEST_IDTR_BASE:
5129 case VMX_VMCS_GUEST_CS_BASE:
5130 case VMX_VMCS_GUEST_DS_BASE:
5131 case VMX_VMCS_GUEST_ES_BASE:
5132 case VMX_VMCS_GUEST_FS_BASE:
5133 case VMX_VMCS_GUEST_GS_BASE:
5134 case VMX_VMCS_GUEST_SS_BASE:
5135 case VMX_VMCS_GUEST_LDTR_BASE:
5136 case VMX_VMCS_GUEST_TR_BASE:
5137 case VMX_VMCS_GUEST_CR3:
5138 return true;
5139 }
5140 return false;
5141}
5142
5143static bool hmR0VmxIsValidReadField(uint32_t idxField)
5144{
5145 switch (idxField)
5146 {
5147 /* Read-only fields. */
5148 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5149 return true;
5150 }
5151 /* Remaining readable fields should also be writable. */
5152 return hmR0VmxIsValidWriteField(idxField);
5153}
5154#endif /* VBOX_STRICT */
5155
5156
5157/**
5158 * Executes the specified handler in 64-bit mode.
5159 *
5160 * @returns VBox status code.
5161 * @param pVM Pointer to the VM.
5162 * @param pVCpu Pointer to the VMCPU.
5163 * @param pCtx Pointer to the guest CPU context.
5164 * @param enmOp The operation to perform.
5165 * @param cbParam Number of parameters.
5166 * @param paParam Array of 32-bit parameters.
5167 */
5168VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5169 uint32_t *paParam)
5170{
5171 int rc, rc2;
5172 PHMGLOBALCPUINFO pCpu;
5173 RTHCPHYS HCPhysCpuPage;
5174 RTCCUINTREG uOldEflags;
5175
5176 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5177 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5178 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5179 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5180
5181#ifdef VBOX_STRICT
5182 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5183 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5184
5185 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5186 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5187#endif
5188
5189 /* Disable interrupts. */
5190 uOldEflags = ASMIntDisableFlags();
5191
5192#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5193 RTCPUID idHostCpu = RTMpCpuId();
5194 CPUMR0SetLApic(pVCpu, idHostCpu);
5195#endif
5196
5197 pCpu = HMR0GetCurrentCpu();
5198 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5199
5200 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5201 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5202
5203 /* Leave VMX Root Mode. */
5204 VMXDisable();
5205
5206 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5207
5208 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5209 CPUMSetHyperEIP(pVCpu, enmOp);
5210 for (int i = (int)cbParam - 1; i >= 0; i--)
5211 CPUMPushHyper(pVCpu, paParam[i]);
5212
5213 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5214
5215 /* Call the switcher. */
5216 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5217 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5218
5219 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5220 /* Make sure the VMX instructions don't cause #UD faults. */
5221 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5222
5223 /* Re-enter VMX Root Mode */
5224 rc2 = VMXEnable(HCPhysCpuPage);
5225 if (RT_FAILURE(rc2))
5226 {
5227 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5228 ASMSetFlags(uOldEflags);
5229 return rc2;
5230 }
5231
5232 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5233 AssertRC(rc2);
5234 Assert(!(ASMGetFlags() & X86_EFL_IF));
5235 ASMSetFlags(uOldEflags);
5236 return rc;
5237}
5238
5239
5240/**
5241 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5242 * supporting 64-bit guests.
5243 *
5244 * @returns VBox status code.
5245 * @param fResume Whether to VMLAUNCH or VMRESUME.
5246 * @param pCtx Pointer to the guest-CPU context.
5247 * @param pCache Pointer to the VMCS cache.
5248 * @param pVM Pointer to the VM.
5249 * @param pVCpu Pointer to the VMCPU.
5250 */
5251DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5252{
5253 uint32_t aParam[6];
5254 PHMGLOBALCPUINFO pCpu = NULL;
5255 RTHCPHYS HCPhysCpuPage = 0;
5256 int rc = VERR_INTERNAL_ERROR_5;
5257
5258 pCpu = HMR0GetCurrentCpu();
5259 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5260
5261#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5262 pCache->uPos = 1;
5263 pCache->interPD = PGMGetInterPaeCR3(pVM);
5264 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5265#endif
5266
5267#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5268 pCache->TestIn.HCPhysCpuPage = 0;
5269 pCache->TestIn.HCPhysVmcs = 0;
5270 pCache->TestIn.pCache = 0;
5271 pCache->TestOut.HCPhysVmcs = 0;
5272 pCache->TestOut.pCache = 0;
5273 pCache->TestOut.pCtx = 0;
5274 pCache->TestOut.eflags = 0;
5275#endif
5276
5277 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5278 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5279 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5280 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5281 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5282 aParam[5] = 0;
5283
5284#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5285 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5286 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5287#endif
5288 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5289
5290#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5291 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5292 Assert(pCtx->dr[4] == 10);
5293 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5294#endif
5295
5296#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5297 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5298 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5299 pVCpu->hm.s.vmx.HCPhysVmcs));
5300 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5301 pCache->TestOut.HCPhysVmcs));
5302 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5303 pCache->TestOut.pCache));
5304 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5305 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5306 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5307 pCache->TestOut.pCtx));
5308 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5309#endif
5310 return rc;
5311}
5312
5313
5314/**
5315 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5316 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5317 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5318 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5319 *
5320 * @returns VBox status code.
5321 * @param pVM Pointer to the VM.
5322 * @param pVCpu Pointer to the VMCPU.
5323 */
5324static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5325{
5326#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5327{ \
5328 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5329 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5330 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5331 ++cReadFields; \
5332}
5333
5334 AssertPtr(pVM);
5335 AssertPtr(pVCpu);
5336 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5337 uint32_t cReadFields = 0;
5338
5339 /*
5340 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5341 * and serve to indicate exceptions to the rules.
5342 */
5343
5344 /* Guest-natural selector base fields. */
5345#if 0
5346 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5347 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5348 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5349#endif
5350 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5351 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5352 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5354 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5362#if 0
5363 /* Unused natural width guest-state fields. */
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5366#endif
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5369
5370 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5371#if 0
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5381#endif
5382
5383 /* Natural width guest-state fields. */
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5385#if 0
5386 /* Currently unused field. */
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5388#endif
5389
5390 if (pVM->hm.s.fNestedPaging)
5391 {
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5393 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5394 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5395 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5396 }
5397 else
5398 {
5399 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5400 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5401 }
5402
5403#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5404 return VINF_SUCCESS;
5405}
5406
5407
5408/**
5409 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5410 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5411 * darwin, running 64-bit guests).
5412 *
5413 * @returns VBox status code.
5414 * @param pVCpu Pointer to the VMCPU.
5415 * @param idxField The VMCS field encoding.
5416 * @param u64Val 16, 32 or 64-bit value.
5417 */
5418VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5419{
5420 int rc;
5421 switch (idxField)
5422 {
5423 /*
5424 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5425 */
5426 /* 64-bit Control fields. */
5427 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5428 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5429 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5430 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5431 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5432 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5433 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5434 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5435 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5436 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5437 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5438 case VMX_VMCS64_CTRL_EPTP_FULL:
5439 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5440 /* 64-bit Guest-state fields. */
5441 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5442 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5443 case VMX_VMCS64_GUEST_PAT_FULL:
5444 case VMX_VMCS64_GUEST_EFER_FULL:
5445 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5446 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5447 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5448 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5449 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5450 /* 64-bit Host-state fields. */
5451 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5452 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5453 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5454 {
5455 rc = VMXWriteVmcs32(idxField, u64Val);
5456 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5457 break;
5458 }
5459
5460 /*
5461 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5462 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5463 */
5464 /* Natural-width Guest-state fields. */
5465 case VMX_VMCS_GUEST_CR3:
5466 case VMX_VMCS_GUEST_ES_BASE:
5467 case VMX_VMCS_GUEST_CS_BASE:
5468 case VMX_VMCS_GUEST_SS_BASE:
5469 case VMX_VMCS_GUEST_DS_BASE:
5470 case VMX_VMCS_GUEST_FS_BASE:
5471 case VMX_VMCS_GUEST_GS_BASE:
5472 case VMX_VMCS_GUEST_LDTR_BASE:
5473 case VMX_VMCS_GUEST_TR_BASE:
5474 case VMX_VMCS_GUEST_GDTR_BASE:
5475 case VMX_VMCS_GUEST_IDTR_BASE:
5476 case VMX_VMCS_GUEST_RSP:
5477 case VMX_VMCS_GUEST_RIP:
5478 case VMX_VMCS_GUEST_SYSENTER_ESP:
5479 case VMX_VMCS_GUEST_SYSENTER_EIP:
5480 {
5481 if (!(u64Val >> 32))
5482 {
5483 /* If this field is 64-bit, VT-x will zero out the top bits. */
5484 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5485 }
5486 else
5487 {
5488 /* Assert that only the 32->64 switcher case should ever come here. */
5489 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5490 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5491 }
5492 break;
5493 }
5494
5495 default:
5496 {
5497 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5498 rc = VERR_INVALID_PARAMETER;
5499 break;
5500 }
5501 }
5502 AssertRCReturn(rc, rc);
5503 return rc;
5504}
5505
5506
5507/**
5508 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5509 * hosts (except darwin) for 64-bit guests.
5510 *
5511 * @param pVCpu Pointer to the VMCPU.
5512 * @param idxField The VMCS field encoding.
5513 * @param u64Val 16, 32 or 64-bit value.
5514 */
5515VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5516{
5517 AssertPtr(pVCpu);
5518 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5519
5520 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5521 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5522
5523 /* Make sure there are no duplicates. */
5524 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5525 {
5526 if (pCache->Write.aField[i] == idxField)
5527 {
5528 pCache->Write.aFieldVal[i] = u64Val;
5529 return VINF_SUCCESS;
5530 }
5531 }
5532
5533 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5534 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5535 pCache->Write.cValidEntries++;
5536 return VINF_SUCCESS;
5537}
5538
5539/* Enable later when the assembly code uses these as callbacks. */
5540#if 0
5541/*
5542 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5543 *
5544 * @param pVCpu Pointer to the VMCPU.
5545 * @param pCache Pointer to the VMCS cache.
5546 *
5547 * @remarks No-long-jump zone!!!
5548 */
5549VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5550{
5551 AssertPtr(pCache);
5552 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5553 {
5554 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5555 AssertRC(rc);
5556 }
5557 pCache->Write.cValidEntries = 0;
5558}
5559
5560
5561/**
5562 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5563 *
5564 * @param pVCpu Pointer to the VMCPU.
5565 * @param pCache Pointer to the VMCS cache.
5566 *
5567 * @remarks No-long-jump zone!!!
5568 */
5569VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5570{
5571 AssertPtr(pCache);
5572 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5573 {
5574 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5575 AssertRC(rc);
5576 }
5577}
5578#endif
5579#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5580
5581
5582/**
5583 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5584 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5585 * timer.
5586 *
5587 * @returns VBox status code.
5588 * @param pVCpu Pointer to the VMCPU.
5589 *
5590 * @remarks No-long-jump zone!!!
5591 */
5592static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5593{
5594 int rc = VERR_INTERNAL_ERROR_5;
5595 bool fOffsettedTsc = false;
5596 bool fParavirtTsc = false;
5597 PVM pVM = pVCpu->CTX_SUFF(pVM);
5598 if (pVM->hm.s.vmx.fUsePreemptTimer)
5599 {
5600 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5601 &pVCpu->hm.s.vmx.u64TSCOffset);
5602
5603 /* Make sure the returned values have sane upper and lower boundaries. */
5604 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5605 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5606 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5607 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5608
5609 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5610 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5611 }
5612 else
5613 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5614
5615#if 1
5616 if (fParavirtTsc)
5617 {
5618#if 1
5619 uint64_t const u64CurTsc = ASMReadTSC();
5620 uint64_t const u64LastTick = TMCpuTickGetLastSeen(pVCpu);
5621 if (u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset < u64LastTick)
5622 {
5623 pVCpu->hm.s.vmx.u64TSCOffset = (u64LastTick - u64CurTsc);
5624 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffsetAdjusted);
5625 }
5626
5627 Assert(u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset >= u64LastTick);
5628#endif
5629 rc = GIMR0UpdateParavirtTsc(pVM, pVCpu->hm.s.vmx.u64TSCOffset);
5630 AssertRC(rc);
5631 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5632 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRC(rc);
5633
5634 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5635 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5636 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5637 }
5638 else
5639#else
5640 if (fParavirtTsc)
5641 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5642#endif
5643 if (fOffsettedTsc)
5644 {
5645 uint64_t u64CurTSC = ASMReadTSC();
5646 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5647 {
5648 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5649 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5650
5651 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5652 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5653 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5654 }
5655 else
5656 {
5657 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5658 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5659 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5660 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5661 }
5662 }
5663 else
5664 {
5665 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5666 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5667 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5668 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5669 }
5670}
5671
5672
5673/**
5674 * Determines if an exception is a contributory exception. Contributory
5675 * exceptions are ones which can cause double-faults. Page-fault is
5676 * intentionally not included here as it's a conditional contributory exception.
5677 *
5678 * @returns true if the exception is contributory, false otherwise.
5679 * @param uVector The exception vector.
5680 */
5681DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5682{
5683 switch (uVector)
5684 {
5685 case X86_XCPT_GP:
5686 case X86_XCPT_SS:
5687 case X86_XCPT_NP:
5688 case X86_XCPT_TS:
5689 case X86_XCPT_DE:
5690 return true;
5691 default:
5692 break;
5693 }
5694 return false;
5695}
5696
5697
5698/**
5699 * Sets an event as a pending event to be injected into the guest.
5700 *
5701 * @param pVCpu Pointer to the VMCPU.
5702 * @param u32IntInfo The VM-entry interruption-information field.
5703 * @param cbInstr The VM-entry instruction length in bytes (for software
5704 * interrupts, exceptions and privileged software
5705 * exceptions).
5706 * @param u32ErrCode The VM-entry exception error code.
5707 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5708 * page-fault.
5709 *
5710 * @remarks Statistics counter assumes this is a guest event being injected or
5711 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5712 * always incremented.
5713 */
5714DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5715 RTGCUINTPTR GCPtrFaultAddress)
5716{
5717 Assert(!pVCpu->hm.s.Event.fPending);
5718 pVCpu->hm.s.Event.fPending = true;
5719 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5720 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5721 pVCpu->hm.s.Event.cbInstr = cbInstr;
5722 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5723
5724 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5725}
5726
5727
5728/**
5729 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5730 *
5731 * @param pVCpu Pointer to the VMCPU.
5732 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5733 * out-of-sync. Make sure to update the required fields
5734 * before using them.
5735 */
5736DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5737{
5738 NOREF(pMixedCtx);
5739 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5740 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5741 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5742 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5743}
5744
5745
5746/**
5747 * Handle a condition that occurred while delivering an event through the guest
5748 * IDT.
5749 *
5750 * @returns VBox status code (informational error codes included).
5751 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5752 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5753 * continue execution of the guest which will delivery the #DF.
5754 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5755 *
5756 * @param pVCpu Pointer to the VMCPU.
5757 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5758 * out-of-sync. Make sure to update the required fields
5759 * before using them.
5760 * @param pVmxTransient Pointer to the VMX transient structure.
5761 *
5762 * @remarks No-long-jump zone!!!
5763 */
5764static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5765{
5766 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5767
5768 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5769 AssertRCReturn(rc, rc);
5770 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5771 {
5772 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5773 AssertRCReturn(rc, rc);
5774
5775 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5776 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5777
5778 typedef enum
5779 {
5780 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5781 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5782 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5783 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5784 } VMXREFLECTXCPT;
5785
5786 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5787 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5788 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5789 {
5790 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5791 {
5792 enmReflect = VMXREFLECTXCPT_XCPT;
5793#ifdef VBOX_STRICT
5794 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5795 && uExitVector == X86_XCPT_PF)
5796 {
5797 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5798 }
5799#endif
5800 if ( uExitVector == X86_XCPT_PF
5801 && uIdtVector == X86_XCPT_PF)
5802 {
5803 pVmxTransient->fVectoringPF = true;
5804 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5805 }
5806 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5807 && hmR0VmxIsContributoryXcpt(uExitVector)
5808 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5809 || uIdtVector == X86_XCPT_PF))
5810 {
5811 enmReflect = VMXREFLECTXCPT_DF;
5812 }
5813 else if (uIdtVector == X86_XCPT_DF)
5814 enmReflect = VMXREFLECTXCPT_TF;
5815 }
5816 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5817 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5818 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5819 {
5820 /*
5821 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5822 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5823 */
5824 enmReflect = VMXREFLECTXCPT_XCPT;
5825 }
5826 }
5827 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5828 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5829 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5830 {
5831 /*
5832 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5833 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5834 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5835 */
5836 enmReflect = VMXREFLECTXCPT_XCPT;
5837 }
5838
5839 /*
5840 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5841 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5842 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5843 *
5844 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5845 */
5846 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5847 && enmReflect == VMXREFLECTXCPT_XCPT
5848 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5849 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5850 {
5851 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5852 }
5853
5854 switch (enmReflect)
5855 {
5856 case VMXREFLECTXCPT_XCPT:
5857 {
5858 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5859 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5860 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5861
5862 uint32_t u32ErrCode = 0;
5863 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5864 {
5865 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5866 AssertRCReturn(rc, rc);
5867 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5868 }
5869
5870 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5871 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5872 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5873 rc = VINF_SUCCESS;
5874 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5875 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5876
5877 break;
5878 }
5879
5880 case VMXREFLECTXCPT_DF:
5881 {
5882 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5883 rc = VINF_HM_DOUBLE_FAULT;
5884 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5885 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5886
5887 break;
5888 }
5889
5890 case VMXREFLECTXCPT_TF:
5891 {
5892 rc = VINF_EM_RESET;
5893 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5894 uExitVector));
5895 break;
5896 }
5897
5898 default:
5899 Assert(rc == VINF_SUCCESS);
5900 break;
5901 }
5902 }
5903 else if ( VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5904 && uExitVector != X86_XCPT_DF
5905 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5906 {
5907 /*
5908 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5909 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5910 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5911 */
5912 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5913 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5914 }
5915
5916 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5917 return rc;
5918}
5919
5920
5921/**
5922 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5923 *
5924 * @returns VBox status code.
5925 * @param pVCpu Pointer to the VMCPU.
5926 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5927 * out-of-sync. Make sure to update the required fields
5928 * before using them.
5929 *
5930 * @remarks No-long-jump zone!!!
5931 */
5932static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5933{
5934 NOREF(pMixedCtx);
5935
5936 /*
5937 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5938 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5939 */
5940 VMMRZCallRing3Disable(pVCpu);
5941 HM_DISABLE_PREEMPT_IF_NEEDED();
5942
5943 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5944 {
5945 uint32_t uVal = 0;
5946 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5947 AssertRCReturn(rc, rc);
5948
5949 uint32_t uShadow = 0;
5950 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5951 AssertRCReturn(rc, rc);
5952
5953 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5954 CPUMSetGuestCR0(pVCpu, uVal);
5955 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5956 }
5957
5958 HM_RESTORE_PREEMPT_IF_NEEDED();
5959 VMMRZCallRing3Enable(pVCpu);
5960 return VINF_SUCCESS;
5961}
5962
5963
5964/**
5965 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5966 *
5967 * @returns VBox status code.
5968 * @param pVCpu Pointer to the VMCPU.
5969 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5970 * out-of-sync. Make sure to update the required fields
5971 * before using them.
5972 *
5973 * @remarks No-long-jump zone!!!
5974 */
5975static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5976{
5977 NOREF(pMixedCtx);
5978
5979 int rc = VINF_SUCCESS;
5980 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5981 {
5982 uint32_t uVal = 0;
5983 uint32_t uShadow = 0;
5984 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5985 AssertRCReturn(rc, rc);
5986 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5987 AssertRCReturn(rc, rc);
5988
5989 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5990 CPUMSetGuestCR4(pVCpu, uVal);
5991 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5992 }
5993 return rc;
5994}
5995
5996
5997/**
5998 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5999 *
6000 * @returns VBox status code.
6001 * @param pVCpu Pointer to the VMCPU.
6002 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6003 * out-of-sync. Make sure to update the required fields
6004 * before using them.
6005 *
6006 * @remarks No-long-jump zone!!!
6007 */
6008static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6009{
6010 int rc = VINF_SUCCESS;
6011 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6012 {
6013 uint64_t u64Val = 0;
6014 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6015 AssertRCReturn(rc, rc);
6016
6017 pMixedCtx->rip = u64Val;
6018 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6019 }
6020 return rc;
6021}
6022
6023
6024/**
6025 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6026 *
6027 * @returns VBox status code.
6028 * @param pVCpu Pointer to the VMCPU.
6029 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6030 * out-of-sync. Make sure to update the required fields
6031 * before using them.
6032 *
6033 * @remarks No-long-jump zone!!!
6034 */
6035static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6036{
6037 int rc = VINF_SUCCESS;
6038 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6039 {
6040 uint64_t u64Val = 0;
6041 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6042 AssertRCReturn(rc, rc);
6043
6044 pMixedCtx->rsp = u64Val;
6045 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6046 }
6047 return rc;
6048}
6049
6050
6051/**
6052 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6053 *
6054 * @returns VBox status code.
6055 * @param pVCpu Pointer to the VMCPU.
6056 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6057 * out-of-sync. Make sure to update the required fields
6058 * before using them.
6059 *
6060 * @remarks No-long-jump zone!!!
6061 */
6062static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6063{
6064 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6065 {
6066 uint32_t uVal = 0;
6067 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6068 AssertRCReturn(rc, rc);
6069
6070 pMixedCtx->eflags.u32 = uVal;
6071 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6072 {
6073 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6074 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6075
6076 pMixedCtx->eflags.Bits.u1VM = 0;
6077 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6078 }
6079
6080 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6081 }
6082 return VINF_SUCCESS;
6083}
6084
6085
6086/**
6087 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6088 * guest-CPU context.
6089 */
6090DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6091{
6092 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6093 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6094 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6095 return rc;
6096}
6097
6098
6099/**
6100 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6101 * from the guest-state area in the VMCS.
6102 *
6103 * @param pVCpu Pointer to the VMCPU.
6104 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6105 * out-of-sync. Make sure to update the required fields
6106 * before using them.
6107 *
6108 * @remarks No-long-jump zone!!!
6109 */
6110static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6111{
6112 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6113 {
6114 uint32_t uIntrState = 0;
6115 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6116 AssertRC(rc);
6117
6118 if (!uIntrState)
6119 {
6120 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6121 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6122
6123 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6124 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6125 }
6126 else
6127 {
6128 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6129 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6130 {
6131 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6132 AssertRC(rc);
6133 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6134 AssertRC(rc);
6135
6136 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6137 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6138 }
6139 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6140 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6141
6142 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6143 {
6144 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6145 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6146 }
6147 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6148 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6149 }
6150
6151 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6152 }
6153}
6154
6155
6156/**
6157 * Saves the guest's activity state.
6158 *
6159 * @returns VBox status code.
6160 * @param pVCpu Pointer to the VMCPU.
6161 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6162 * out-of-sync. Make sure to update the required fields
6163 * before using them.
6164 *
6165 * @remarks No-long-jump zone!!!
6166 */
6167static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6168{
6169 NOREF(pMixedCtx);
6170 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6171 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6172 return VINF_SUCCESS;
6173}
6174
6175
6176/**
6177 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6178 * the current VMCS into the guest-CPU context.
6179 *
6180 * @returns VBox status code.
6181 * @param pVCpu Pointer to the VMCPU.
6182 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6183 * out-of-sync. Make sure to update the required fields
6184 * before using them.
6185 *
6186 * @remarks No-long-jump zone!!!
6187 */
6188static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6189{
6190 int rc = VINF_SUCCESS;
6191 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6192 {
6193 uint32_t u32Val = 0;
6194 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6195 pMixedCtx->SysEnter.cs = u32Val;
6196 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6197 }
6198
6199 uint64_t u64Val = 0;
6200 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6201 {
6202 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6203 pMixedCtx->SysEnter.eip = u64Val;
6204 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6205 }
6206 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6207 {
6208 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6209 pMixedCtx->SysEnter.esp = u64Val;
6210 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6211 }
6212 return rc;
6213}
6214
6215
6216/**
6217 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6218 * the CPU back into the guest-CPU context.
6219 *
6220 * @returns VBox status code.
6221 * @param pVCpu Pointer to the VMCPU.
6222 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6223 * out-of-sync. Make sure to update the required fields
6224 * before using them.
6225 *
6226 * @remarks No-long-jump zone!!!
6227 */
6228static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6229{
6230#if HC_ARCH_BITS == 64
6231 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6232 {
6233 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6234 VMMRZCallRing3Disable(pVCpu);
6235 HM_DISABLE_PREEMPT_IF_NEEDED();
6236
6237 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6238 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6239 {
6240 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6241 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6242 }
6243
6244 HM_RESTORE_PREEMPT_IF_NEEDED();
6245 VMMRZCallRing3Enable(pVCpu);
6246 }
6247 else
6248 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6249#else
6250 NOREF(pMixedCtx);
6251 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6252#endif
6253
6254 return VINF_SUCCESS;
6255}
6256
6257
6258/**
6259 * Saves the auto load/store'd guest MSRs from the current VMCS into
6260 * the guest-CPU context.
6261 *
6262 * @returns VBox status code.
6263 * @param pVCpu Pointer to the VMCPU.
6264 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6265 * out-of-sync. Make sure to update the required fields
6266 * before using them.
6267 *
6268 * @remarks No-long-jump zone!!!
6269 */
6270static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6271{
6272 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6273 return VINF_SUCCESS;
6274
6275 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6276 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6277 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6278 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6279 {
6280 switch (pMsr->u32Msr)
6281 {
6282 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6283 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6284 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6285 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6286 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6287 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6288 break;
6289
6290 default:
6291 {
6292 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6293 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6294 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6295 }
6296 }
6297 }
6298
6299 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6300 return VINF_SUCCESS;
6301}
6302
6303
6304/**
6305 * Saves the guest control registers from the current VMCS into the guest-CPU
6306 * context.
6307 *
6308 * @returns VBox status code.
6309 * @param pVCpu Pointer to the VMCPU.
6310 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6311 * out-of-sync. Make sure to update the required fields
6312 * before using them.
6313 *
6314 * @remarks No-long-jump zone!!!
6315 */
6316static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6317{
6318 /* Guest CR0. Guest FPU. */
6319 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6320 AssertRCReturn(rc, rc);
6321
6322 /* Guest CR4. */
6323 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6324 AssertRCReturn(rc, rc);
6325
6326 /* Guest CR2 - updated always during the world-switch or in #PF. */
6327 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6328 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6329 {
6330 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6331 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6332
6333 PVM pVM = pVCpu->CTX_SUFF(pVM);
6334 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6335 || ( pVM->hm.s.fNestedPaging
6336 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6337 {
6338 uint64_t u64Val = 0;
6339 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6340 if (pMixedCtx->cr3 != u64Val)
6341 {
6342 CPUMSetGuestCR3(pVCpu, u64Val);
6343 if (VMMRZCallRing3IsEnabled(pVCpu))
6344 {
6345 PGMUpdateCR3(pVCpu, u64Val);
6346 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6347 }
6348 else
6349 {
6350 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6351 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6352 }
6353 }
6354
6355 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6356 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6357 {
6358 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6359 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6360 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6361 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6362
6363 if (VMMRZCallRing3IsEnabled(pVCpu))
6364 {
6365 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6366 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6367 }
6368 else
6369 {
6370 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6371 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6372 }
6373 }
6374 }
6375
6376 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6377 }
6378
6379 /*
6380 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6381 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6382 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6383 *
6384 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6385 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6386 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6387 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6388 *
6389 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6390 */
6391 if (VMMRZCallRing3IsEnabled(pVCpu))
6392 {
6393 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6394 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6395
6396 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6397 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6398
6399 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6400 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6401 }
6402
6403 return rc;
6404}
6405
6406
6407/**
6408 * Reads a guest segment register from the current VMCS into the guest-CPU
6409 * context.
6410 *
6411 * @returns VBox status code.
6412 * @param pVCpu Pointer to the VMCPU.
6413 * @param idxSel Index of the selector in the VMCS.
6414 * @param idxLimit Index of the segment limit in the VMCS.
6415 * @param idxBase Index of the segment base in the VMCS.
6416 * @param idxAccess Index of the access rights of the segment in the VMCS.
6417 * @param pSelReg Pointer to the segment selector.
6418 *
6419 * @remarks No-long-jump zone!!!
6420 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6421 * macro as that takes care of whether to read from the VMCS cache or
6422 * not.
6423 */
6424DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6425 PCPUMSELREG pSelReg)
6426{
6427 NOREF(pVCpu);
6428
6429 uint32_t u32Val = 0;
6430 int rc = VMXReadVmcs32(idxSel, &u32Val);
6431 AssertRCReturn(rc, rc);
6432 pSelReg->Sel = (uint16_t)u32Val;
6433 pSelReg->ValidSel = (uint16_t)u32Val;
6434 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6435
6436 rc = VMXReadVmcs32(idxLimit, &u32Val);
6437 AssertRCReturn(rc, rc);
6438 pSelReg->u32Limit = u32Val;
6439
6440 uint64_t u64Val = 0;
6441 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6442 AssertRCReturn(rc, rc);
6443 pSelReg->u64Base = u64Val;
6444
6445 rc = VMXReadVmcs32(idxAccess, &u32Val);
6446 AssertRCReturn(rc, rc);
6447 pSelReg->Attr.u = u32Val;
6448
6449 /*
6450 * If VT-x marks the segment as unusable, most other bits remain undefined:
6451 * - For CS the L, D and G bits have meaning.
6452 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6453 * - For the remaining data segments no bits are defined.
6454 *
6455 * The present bit and the unusable bit has been observed to be set at the
6456 * same time (the selector was supposed to be invalid as we started executing
6457 * a V8086 interrupt in ring-0).
6458 *
6459 * What should be important for the rest of the VBox code, is that the P bit is
6460 * cleared. Some of the other VBox code recognizes the unusable bit, but
6461 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6462 * safe side here, we'll strip off P and other bits we don't care about. If
6463 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6464 *
6465 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6466 */
6467 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6468 {
6469 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6470
6471 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6472 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6473 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6474
6475 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6476#ifdef DEBUG_bird
6477 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6478 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6479 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6480#endif
6481 }
6482 return VINF_SUCCESS;
6483}
6484
6485
6486#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6487# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6488 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6489 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6490#else
6491# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6492 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6493 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6494#endif
6495
6496
6497/**
6498 * Saves the guest segment registers from the current VMCS into the guest-CPU
6499 * context.
6500 *
6501 * @returns VBox status code.
6502 * @param pVCpu Pointer to the VMCPU.
6503 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6504 * out-of-sync. Make sure to update the required fields
6505 * before using them.
6506 *
6507 * @remarks No-long-jump zone!!!
6508 */
6509static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6510{
6511 /* Guest segment registers. */
6512 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6513 {
6514 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6515 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6516 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6517 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6518 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6519 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6520 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6521
6522 /* Restore segment attributes for real-on-v86 mode hack. */
6523 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6524 {
6525 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6526 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6527 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6528 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6529 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6530 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6531 }
6532 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6533 }
6534
6535 return VINF_SUCCESS;
6536}
6537
6538
6539/**
6540 * Saves the guest descriptor table registers and task register from the current
6541 * VMCS into the guest-CPU context.
6542 *
6543 * @returns VBox status code.
6544 * @param pVCpu Pointer to the VMCPU.
6545 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6546 * out-of-sync. Make sure to update the required fields
6547 * before using them.
6548 *
6549 * @remarks No-long-jump zone!!!
6550 */
6551static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6552{
6553 int rc = VINF_SUCCESS;
6554
6555 /* Guest LDTR. */
6556 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6557 {
6558 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6559 AssertRCReturn(rc, rc);
6560 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6561 }
6562
6563 /* Guest GDTR. */
6564 uint64_t u64Val = 0;
6565 uint32_t u32Val = 0;
6566 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6567 {
6568 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6569 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6570 pMixedCtx->gdtr.pGdt = u64Val;
6571 pMixedCtx->gdtr.cbGdt = u32Val;
6572 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6573 }
6574
6575 /* Guest IDTR. */
6576 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6577 {
6578 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6579 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6580 pMixedCtx->idtr.pIdt = u64Val;
6581 pMixedCtx->idtr.cbIdt = u32Val;
6582 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6583 }
6584
6585 /* Guest TR. */
6586 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6587 {
6588 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6589 AssertRCReturn(rc, rc);
6590
6591 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6592 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6593 {
6594 rc = VMXLOCAL_READ_SEG(TR, tr);
6595 AssertRCReturn(rc, rc);
6596 }
6597 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6598 }
6599 return rc;
6600}
6601
6602#undef VMXLOCAL_READ_SEG
6603
6604
6605/**
6606 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6607 * context.
6608 *
6609 * @returns VBox status code.
6610 * @param pVCpu Pointer to the VMCPU.
6611 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6612 * out-of-sync. Make sure to update the required fields
6613 * before using them.
6614 *
6615 * @remarks No-long-jump zone!!!
6616 */
6617static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6618{
6619 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6620 {
6621 if (!pVCpu->hm.s.fUsingHyperDR7)
6622 {
6623 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6624 uint32_t u32Val;
6625 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6626 pMixedCtx->dr[7] = u32Val;
6627 }
6628
6629 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6630 }
6631 return VINF_SUCCESS;
6632}
6633
6634
6635/**
6636 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6637 *
6638 * @returns VBox status code.
6639 * @param pVCpu Pointer to the VMCPU.
6640 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6641 * out-of-sync. Make sure to update the required fields
6642 * before using them.
6643 *
6644 * @remarks No-long-jump zone!!!
6645 */
6646static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6647{
6648 NOREF(pMixedCtx);
6649
6650 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6651 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6652 return VINF_SUCCESS;
6653}
6654
6655
6656/**
6657 * Saves the entire guest state from the currently active VMCS into the
6658 * guest-CPU context. This essentially VMREADs all guest-data.
6659 *
6660 * @returns VBox status code.
6661 * @param pVCpu Pointer to the VMCPU.
6662 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6663 * out-of-sync. Make sure to update the required fields
6664 * before using them.
6665 */
6666static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6667{
6668 Assert(pVCpu);
6669 Assert(pMixedCtx);
6670
6671 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6672 return VINF_SUCCESS;
6673
6674 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6675 again on the ring-3 callback path, there is no real need to. */
6676 if (VMMRZCallRing3IsEnabled(pVCpu))
6677 VMMR0LogFlushDisable(pVCpu);
6678 else
6679 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6680 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6681
6682 /* In case we get preempted before saving the interruptibility-state in hmR0VmxPostRunGuest(), do it here.
6683 Otherwise we lose the info. from the VMCS if we get rescheduled on a different host CPU. */
6684 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6685
6686 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6687 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6688
6689 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6690 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6691
6692 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6693 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6694
6695 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6696 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6697
6698 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6699 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6700
6701 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6702 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6703
6704 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6705 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6706
6707 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6708 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6709
6710 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6711 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6712
6713 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6714 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6715
6716 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6717 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6718
6719 if (VMMRZCallRing3IsEnabled(pVCpu))
6720 VMMR0LogFlushEnable(pVCpu);
6721
6722 return rc;
6723}
6724
6725
6726/**
6727 * Check per-VM and per-VCPU force flag actions that require us to go back to
6728 * ring-3 for one reason or another.
6729 *
6730 * @returns VBox status code (information status code included).
6731 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6732 * ring-3.
6733 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6734 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6735 * interrupts)
6736 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6737 * all EMTs to be in ring-3.
6738 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6739 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6740 * to the EM loop.
6741 *
6742 * @param pVM Pointer to the VM.
6743 * @param pVCpu Pointer to the VMCPU.
6744 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6745 * out-of-sync. Make sure to update the required fields
6746 * before using them.
6747 */
6748static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6749{
6750 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6751
6752 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6753 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6754 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6755 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6756 {
6757 /* We need the control registers now, make sure the guest-CPU context is updated. */
6758 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6759 AssertRCReturn(rc3, rc3);
6760
6761 /* Pending HM CR3 sync. */
6762 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6763 {
6764 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6765 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6766 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6767 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6768 }
6769
6770 /* Pending HM PAE PDPEs. */
6771 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6772 {
6773 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6774 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6775 }
6776
6777 /* Pending PGM C3 sync. */
6778 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6779 {
6780 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6781 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6782 if (rc2 != VINF_SUCCESS)
6783 {
6784 AssertRC(rc2);
6785 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6786 return rc2;
6787 }
6788 }
6789
6790 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6791 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6792 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6793 {
6794 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6795 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6796 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6797 return rc2;
6798 }
6799
6800 /* Pending VM request packets, such as hardware interrupts. */
6801 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6802 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6803 {
6804 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6805 return VINF_EM_PENDING_REQUEST;
6806 }
6807
6808 /* Pending PGM pool flushes. */
6809 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6810 {
6811 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6812 return VINF_PGM_POOL_FLUSH_PENDING;
6813 }
6814
6815 /* Pending DMA requests. */
6816 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6817 {
6818 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6819 return VINF_EM_RAW_TO_R3;
6820 }
6821 }
6822
6823 return VINF_SUCCESS;
6824}
6825
6826
6827/**
6828 * Converts any TRPM trap into a pending HM event. This is typically used when
6829 * entering from ring-3 (not longjmp returns).
6830 *
6831 * @param pVCpu Pointer to the VMCPU.
6832 */
6833static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6834{
6835 Assert(TRPMHasTrap(pVCpu));
6836 Assert(!pVCpu->hm.s.Event.fPending);
6837
6838 uint8_t uVector;
6839 TRPMEVENT enmTrpmEvent;
6840 RTGCUINT uErrCode;
6841 RTGCUINTPTR GCPtrFaultAddress;
6842 uint8_t cbInstr;
6843
6844 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6845 AssertRC(rc);
6846
6847 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6848 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6849 if (enmTrpmEvent == TRPM_TRAP)
6850 {
6851 switch (uVector)
6852 {
6853 case X86_XCPT_NMI:
6854 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6855 break;
6856
6857 case X86_XCPT_BP:
6858 case X86_XCPT_OF:
6859 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6860 break;
6861
6862 case X86_XCPT_PF:
6863 case X86_XCPT_DF:
6864 case X86_XCPT_TS:
6865 case X86_XCPT_NP:
6866 case X86_XCPT_SS:
6867 case X86_XCPT_GP:
6868 case X86_XCPT_AC:
6869 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6870 /* no break! */
6871 default:
6872 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6873 break;
6874 }
6875 }
6876 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6877 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6878 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6879 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6880 else
6881 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6882
6883 rc = TRPMResetTrap(pVCpu);
6884 AssertRC(rc);
6885 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6886 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6887
6888 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6889 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6890}
6891
6892
6893/**
6894 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6895 * VT-x to execute any instruction.
6896 *
6897 * @param pvCpu Pointer to the VMCPU.
6898 */
6899static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6900{
6901 Assert(pVCpu->hm.s.Event.fPending);
6902
6903 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6904 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6905 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6906 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6907
6908 /* If a trap was already pending, we did something wrong! */
6909 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6910
6911 TRPMEVENT enmTrapType;
6912 switch (uVectorType)
6913 {
6914 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6915 enmTrapType = TRPM_HARDWARE_INT;
6916 break;
6917
6918 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6919 enmTrapType = TRPM_SOFTWARE_INT;
6920 break;
6921
6922 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6923 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6924 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6925 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6926 enmTrapType = TRPM_TRAP;
6927 break;
6928
6929 default:
6930 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6931 enmTrapType = TRPM_32BIT_HACK;
6932 break;
6933 }
6934
6935 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6936
6937 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6938 AssertRC(rc);
6939
6940 if (fErrorCodeValid)
6941 TRPMSetErrorCode(pVCpu, uErrorCode);
6942
6943 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6944 && uVector == X86_XCPT_PF)
6945 {
6946 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6947 }
6948 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6949 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6950 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6951 {
6952 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6953 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6954 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6955 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6956 }
6957 pVCpu->hm.s.Event.fPending = false;
6958}
6959
6960
6961/**
6962 * Does the necessary state syncing before returning to ring-3 for any reason
6963 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6964 *
6965 * @returns VBox status code.
6966 * @param pVM Pointer to the VM.
6967 * @param pVCpu Pointer to the VMCPU.
6968 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6969 * be out-of-sync. Make sure to update the required
6970 * fields before using them.
6971 * @param fSaveGuestState Whether to save the guest state or not.
6972 *
6973 * @remarks No-long-jmp zone!!!
6974 */
6975static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6976{
6977 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6978 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6979
6980 RTCPUID idCpu = RTMpCpuId();
6981 Log4Func(("HostCpuId=%u\n", idCpu));
6982
6983 /*
6984 * !!! IMPORTANT !!!
6985 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6986 */
6987
6988 /* Save the guest state if necessary. */
6989 if ( fSaveGuestState
6990 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6991 {
6992 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6993 AssertRCReturn(rc, rc);
6994 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6995 }
6996
6997 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6998 if (CPUMIsGuestFPUStateActive(pVCpu))
6999 {
7000 /* We shouldn't reload CR0 without saving it first. */
7001 if (!fSaveGuestState)
7002 {
7003 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7004 AssertRCReturn(rc, rc);
7005 }
7006 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7007 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7008 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7009 }
7010
7011 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7012#ifdef VBOX_STRICT
7013 if (CPUMIsHyperDebugStateActive(pVCpu))
7014 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7015#endif
7016 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7017 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7018 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7019 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7020
7021#if HC_ARCH_BITS == 64
7022 /* Restore host-state bits that VT-x only restores partially. */
7023 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7024 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7025 {
7026 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7027 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7028 }
7029 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7030#endif
7031
7032#if HC_ARCH_BITS == 64
7033 /* Restore the host MSRs as we're leaving VT-x context. */
7034 if ( pVM->hm.s.fAllow64BitGuests
7035 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7036 {
7037 /* We shouldn't reload the guest MSRs without saving it first. */
7038 if (!fSaveGuestState)
7039 {
7040 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7041 AssertRCReturn(rc, rc);
7042 }
7043 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7044 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7045 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
7046 }
7047#endif
7048
7049 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7050 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7051
7052 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7053 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7054 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7055 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7056 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7057 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7058 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7059 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7060
7061 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7062
7063 /** @todo This partially defeats the purpose of having preemption hooks.
7064 * The problem is, deregistering the hooks should be moved to a place that
7065 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7066 * context.
7067 */
7068 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7069 {
7070 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7071 AssertRCReturn(rc, rc);
7072
7073 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7074 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7075 }
7076 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7077 NOREF(idCpu);
7078
7079 return VINF_SUCCESS;
7080}
7081
7082
7083/**
7084 * Leaves the VT-x session.
7085 *
7086 * @returns VBox status code.
7087 * @param pVM Pointer to the VM.
7088 * @param pVCpu Pointer to the VMCPU.
7089 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7090 * out-of-sync. Make sure to update the required fields
7091 * before using them.
7092 *
7093 * @remarks No-long-jmp zone!!!
7094 */
7095DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7096{
7097 HM_DISABLE_PREEMPT_IF_NEEDED();
7098 HMVMX_ASSERT_CPU_SAFE();
7099 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7101
7102 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7103 and done this from the VMXR0ThreadCtxCallback(). */
7104 if (!pVCpu->hm.s.fLeaveDone)
7105 {
7106 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7107 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7108 pVCpu->hm.s.fLeaveDone = true;
7109 }
7110 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7111
7112 /*
7113 * !!! IMPORTANT !!!
7114 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7115 */
7116
7117 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7118 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7119 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7120 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7121 VMMR0ThreadCtxHooksDeregister(pVCpu);
7122
7123 /* Leave HM context. This takes care of local init (term). */
7124 int rc = HMR0LeaveCpu(pVCpu);
7125
7126 HM_RESTORE_PREEMPT_IF_NEEDED();
7127
7128 return rc;
7129}
7130
7131
7132/**
7133 * Does the necessary state syncing before doing a longjmp to ring-3.
7134 *
7135 * @returns VBox status code.
7136 * @param pVM Pointer to the VM.
7137 * @param pVCpu Pointer to the VMCPU.
7138 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7139 * out-of-sync. Make sure to update the required fields
7140 * before using them.
7141 *
7142 * @remarks No-long-jmp zone!!!
7143 */
7144DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7145{
7146 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7147}
7148
7149
7150/**
7151 * Take necessary actions before going back to ring-3.
7152 *
7153 * An action requires us to go back to ring-3. This function does the necessary
7154 * steps before we can safely return to ring-3. This is not the same as longjmps
7155 * to ring-3, this is voluntary and prepares the guest so it may continue
7156 * executing outside HM (recompiler/IEM).
7157 *
7158 * @returns VBox status code.
7159 * @param pVM Pointer to the VM.
7160 * @param pVCpu Pointer to the VMCPU.
7161 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7162 * out-of-sync. Make sure to update the required fields
7163 * before using them.
7164 * @param rcExit The reason for exiting to ring-3. Can be
7165 * VINF_VMM_UNKNOWN_RING3_CALL.
7166 */
7167static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7168{
7169 Assert(pVM);
7170 Assert(pVCpu);
7171 Assert(pMixedCtx);
7172 HMVMX_ASSERT_PREEMPT_SAFE();
7173
7174 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7175 {
7176 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7177 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7178 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7179 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7180 }
7181
7182 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7183 VMMRZCallRing3Disable(pVCpu);
7184 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7185
7186 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7187 if (pVCpu->hm.s.Event.fPending)
7188 {
7189 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7190 Assert(!pVCpu->hm.s.Event.fPending);
7191 }
7192
7193 /* Save guest state and restore host state bits. */
7194 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7195 AssertRCReturn(rc, rc);
7196 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7197
7198 /* Sync recompiler state. */
7199 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7200 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7201 | CPUM_CHANGED_LDTR
7202 | CPUM_CHANGED_GDTR
7203 | CPUM_CHANGED_IDTR
7204 | CPUM_CHANGED_TR
7205 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7206 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7207 if ( pVM->hm.s.fNestedPaging
7208 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7209 {
7210 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7211 }
7212
7213 Assert(!pVCpu->hm.s.fClearTrapFlag);
7214
7215 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7216 if (rcExit != VINF_EM_RAW_INTERRUPT)
7217 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7218
7219 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7220
7221 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7222 VMMRZCallRing3RemoveNotification(pVCpu);
7223 VMMRZCallRing3Enable(pVCpu);
7224
7225 return rc;
7226}
7227
7228
7229/**
7230 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7231 * longjump to ring-3 and possibly get preempted.
7232 *
7233 * @returns VBox status code.
7234 * @param pVCpu Pointer to the VMCPU.
7235 * @param enmOperation The operation causing the ring-3 longjump.
7236 * @param pvUser Opaque pointer to the guest-CPU context. The data
7237 * may be out-of-sync. Make sure to update the required
7238 * fields before using them.
7239 */
7240DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7241{
7242 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7243 {
7244 /*
7245 * !!! IMPORTANT !!!
7246 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7247 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7248 */
7249 VMMRZCallRing3RemoveNotification(pVCpu);
7250 VMMRZCallRing3Disable(pVCpu);
7251 HM_DISABLE_PREEMPT_IF_NEEDED();
7252
7253 PVM pVM = pVCpu->CTX_SUFF(pVM);
7254 if (CPUMIsGuestFPUStateActive(pVCpu))
7255 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7256
7257 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7258
7259#if HC_ARCH_BITS == 64
7260 /* Restore host-state bits that VT-x only restores partially. */
7261 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7262 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7263 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7264 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7265
7266 /* Restore the host MSRs as we're leaving VT-x context. */
7267 if ( pVM->hm.s.fAllow64BitGuests
7268 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7269 {
7270 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7271 }
7272#endif
7273 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7274 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7275 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7276 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7277 {
7278 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7279 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7280 }
7281
7282 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7283 VMMR0ThreadCtxHooksDeregister(pVCpu);
7284
7285 HMR0LeaveCpu(pVCpu);
7286 HM_RESTORE_PREEMPT_IF_NEEDED();
7287 return VINF_SUCCESS;
7288 }
7289
7290 Assert(pVCpu);
7291 Assert(pvUser);
7292 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7293 HMVMX_ASSERT_PREEMPT_SAFE();
7294
7295 VMMRZCallRing3Disable(pVCpu);
7296 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7297
7298 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7299 enmOperation));
7300
7301 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7302 AssertRCReturn(rc, rc);
7303
7304 VMMRZCallRing3Enable(pVCpu);
7305 return VINF_SUCCESS;
7306}
7307
7308
7309/**
7310 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7311 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7312 *
7313 * @param pVCpu Pointer to the VMCPU.
7314 */
7315DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7316{
7317 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7318 {
7319 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7320 {
7321 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7322 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7323 AssertRC(rc);
7324 Log4(("Setup interrupt-window exiting\n"));
7325 }
7326 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7327}
7328
7329
7330/**
7331 * Clears the interrupt-window exiting control in the VMCS.
7332 *
7333 * @param pVCpu Pointer to the VMCPU.
7334 */
7335DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7336{
7337 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7338 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7339 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7340 AssertRC(rc);
7341 Log4(("Cleared interrupt-window exiting\n"));
7342}
7343
7344
7345/**
7346 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7347 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7348 *
7349 * @param pVCpu Pointer to the VMCPU.
7350 */
7351DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7352{
7353 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7354 {
7355 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7356 {
7357 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7358 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7359 AssertRC(rc);
7360 Log4(("Setup NMI-window exiting\n"));
7361 }
7362 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7363}
7364
7365
7366/**
7367 * Clears the NMI-window exiting control in the VMCS.
7368 *
7369 * @param pVCpu Pointer to the VMCPU.
7370 */
7371DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7372{
7373 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7374 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7375 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7376 AssertRC(rc);
7377 Log4(("Cleared NMI-window exiting\n"));
7378}
7379
7380
7381/**
7382 * Evaluates the event to be delivered to the guest and sets it as the pending
7383 * event.
7384 *
7385 * @param pVCpu Pointer to the VMCPU.
7386 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7387 * out-of-sync. Make sure to update the required fields
7388 * before using them.
7389 */
7390static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7391{
7392 Assert(!pVCpu->hm.s.Event.fPending);
7393
7394 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7395 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7396 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7397 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7398 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7399
7400 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7401 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7402 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7403 Assert(!TRPMHasTrap(pVCpu));
7404
7405 /*
7406 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7407 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7408 */
7409 /** @todo SMI. SMIs take priority over NMIs. */
7410 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7411 {
7412 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7413 if ( !fBlockNmi
7414 && !fBlockSti
7415 && !fBlockMovSS)
7416 {
7417 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7418 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7419 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7420
7421 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7422 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7423 }
7424 else
7425 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7426 }
7427 /*
7428 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7429 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7430 */
7431 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7432 && !pVCpu->hm.s.fSingleInstruction)
7433 {
7434 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7435 AssertRC(rc);
7436 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7437 if ( !fBlockInt
7438 && !fBlockSti
7439 && !fBlockMovSS)
7440 {
7441 uint8_t u8Interrupt;
7442 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7443 if (RT_SUCCESS(rc))
7444 {
7445 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7446 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7447 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7448
7449 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7450 }
7451 else
7452 {
7453 /** @todo Does this actually happen? If not turn it into an assertion. */
7454 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7455 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7456 }
7457 }
7458 else
7459 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7460 }
7461}
7462
7463
7464/**
7465 * Sets a pending-debug exception to be delivered to the guest if the guest is
7466 * single-stepping.
7467 *
7468 * @param pVCpu Pointer to the VMCPU.
7469 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7470 * out-of-sync. Make sure to update the required fields
7471 * before using them.
7472 */
7473DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7474{
7475 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7476 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7477 {
7478 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7479 AssertRC(rc);
7480 }
7481}
7482
7483
7484/**
7485 * Injects any pending events into the guest if the guest is in a state to
7486 * receive them.
7487 *
7488 * @returns VBox status code (informational status codes included).
7489 * @param pVCpu Pointer to the VMCPU.
7490 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7491 * out-of-sync. Make sure to update the required fields
7492 * before using them.
7493 */
7494static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7495{
7496 HMVMX_ASSERT_PREEMPT_SAFE();
7497 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7498
7499 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7500 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7501 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7502 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7503
7504 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7505 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7506 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7507 Assert(!TRPMHasTrap(pVCpu));
7508
7509 int rc = VINF_SUCCESS;
7510 if (pVCpu->hm.s.Event.fPending)
7511 {
7512 /*
7513 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7514 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7515 * ended up enabling interrupts outside VT-x.
7516 */
7517 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7518 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7519 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7520 {
7521 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7522 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7523 }
7524
7525#ifdef VBOX_STRICT
7526 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7527 {
7528 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7529 Assert(!fBlockInt);
7530 Assert(!fBlockSti);
7531 Assert(!fBlockMovSS);
7532 }
7533 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7534 {
7535 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7536 Assert(!fBlockSti);
7537 Assert(!fBlockMovSS);
7538 Assert(!fBlockNmi);
7539 }
7540#endif
7541 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7542 (uint8_t)uIntType));
7543 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7544 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7545 AssertRCReturn(rc, rc);
7546
7547 /* Update the interruptibility-state as it could have been changed by
7548 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7549 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7550 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7551
7552#ifdef VBOX_WITH_STATISTICS
7553 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7554 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7555 else
7556 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7557#endif
7558 }
7559
7560 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7561 if ( fBlockSti
7562 || fBlockMovSS)
7563 {
7564 if ( !pVCpu->hm.s.fSingleInstruction
7565 && !DBGFIsStepping(pVCpu))
7566 {
7567 /*
7568 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7569 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7570 * See Intel spec. 27.3.4 "Saving Non-Register State".
7571 */
7572 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7573 AssertRCReturn(rc2, rc2);
7574 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7575 }
7576 else if (pMixedCtx->eflags.Bits.u1TF)
7577 {
7578 /*
7579 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7580 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7581 */
7582 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7583 uIntrState = 0;
7584 }
7585 }
7586
7587 /*
7588 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7589 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7590 */
7591 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7592 AssertRC(rc2);
7593
7594 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7595 NOREF(fBlockMovSS); NOREF(fBlockSti);
7596 return rc;
7597}
7598
7599
7600/**
7601 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7602 *
7603 * @param pVCpu Pointer to the VMCPU.
7604 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7605 * out-of-sync. Make sure to update the required fields
7606 * before using them.
7607 */
7608DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7609{
7610 NOREF(pMixedCtx);
7611 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7612 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7613}
7614
7615
7616/**
7617 * Injects a double-fault (#DF) exception into the VM.
7618 *
7619 * @returns VBox status code (informational status code included).
7620 * @param pVCpu Pointer to the VMCPU.
7621 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7622 * out-of-sync. Make sure to update the required fields
7623 * before using them.
7624 */
7625DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7626{
7627 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7628 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7629 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7630 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7631 puIntrState);
7632}
7633
7634
7635/**
7636 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7637 *
7638 * @param pVCpu Pointer to the VMCPU.
7639 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7640 * out-of-sync. Make sure to update the required fields
7641 * before using them.
7642 */
7643DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7644{
7645 NOREF(pMixedCtx);
7646 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7647 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7648 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7649}
7650
7651
7652/**
7653 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7654 *
7655 * @param pVCpu Pointer to the VMCPU.
7656 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7657 * out-of-sync. Make sure to update the required fields
7658 * before using them.
7659 * @param cbInstr The value of RIP that is to be pushed on the guest
7660 * stack.
7661 */
7662DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7663{
7664 NOREF(pMixedCtx);
7665 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7666 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7667 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7668}
7669
7670
7671/**
7672 * Injects a general-protection (#GP) fault into the VM.
7673 *
7674 * @returns VBox status code (informational status code included).
7675 * @param pVCpu Pointer to the VMCPU.
7676 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7677 * out-of-sync. Make sure to update the required fields
7678 * before using them.
7679 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7680 * mode, i.e. in real-mode it's not valid).
7681 * @param u32ErrorCode The error code associated with the #GP.
7682 */
7683DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7684 uint32_t *puIntrState)
7685{
7686 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7687 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7688 if (fErrorCodeValid)
7689 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7690 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7691 puIntrState);
7692}
7693
7694
7695/**
7696 * Sets a general-protection (#GP) exception as pending-for-injection into the
7697 * VM.
7698 *
7699 * @param pVCpu Pointer to the VMCPU.
7700 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7701 * out-of-sync. Make sure to update the required fields
7702 * before using them.
7703 * @param u32ErrorCode The error code associated with the #GP.
7704 */
7705DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7706{
7707 NOREF(pMixedCtx);
7708 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7709 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7710 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7711 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7712}
7713
7714
7715/**
7716 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7717 *
7718 * @param pVCpu Pointer to the VMCPU.
7719 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7720 * out-of-sync. Make sure to update the required fields
7721 * before using them.
7722 * @param uVector The software interrupt vector number.
7723 * @param cbInstr The value of RIP that is to be pushed on the guest
7724 * stack.
7725 */
7726DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7727{
7728 NOREF(pMixedCtx);
7729 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7730 if ( uVector == X86_XCPT_BP
7731 || uVector == X86_XCPT_OF)
7732 {
7733 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7734 }
7735 else
7736 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7737 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7738}
7739
7740
7741/**
7742 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7743 * stack.
7744 *
7745 * @returns VBox status code (information status code included).
7746 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7747 * @param pVM Pointer to the VM.
7748 * @param pMixedCtx Pointer to the guest-CPU context.
7749 * @param uValue The value to push to the guest stack.
7750 */
7751DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7752{
7753 /*
7754 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7755 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7756 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7757 */
7758 if (pMixedCtx->sp == 1)
7759 return VINF_EM_RESET;
7760 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7761 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7762 AssertRCReturn(rc, rc);
7763 return rc;
7764}
7765
7766
7767/**
7768 * Injects an event into the guest upon VM-entry by updating the relevant fields
7769 * in the VM-entry area in the VMCS.
7770 *
7771 * @returns VBox status code (informational error codes included).
7772 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7773 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7774 *
7775 * @param pVCpu Pointer to the VMCPU.
7776 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7777 * be out-of-sync. Make sure to update the required
7778 * fields before using them.
7779 * @param u64IntInfo The VM-entry interruption-information field.
7780 * @param cbInstr The VM-entry instruction length in bytes (for
7781 * software interrupts, exceptions and privileged
7782 * software exceptions).
7783 * @param u32ErrCode The VM-entry exception error code.
7784 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7785 * @param puIntrState Pointer to the current guest interruptibility-state.
7786 * This interruptibility-state will be updated if
7787 * necessary. This cannot not be NULL.
7788 *
7789 * @remarks Requires CR0!
7790 * @remarks No-long-jump zone!!!
7791 */
7792static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7793 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7794{
7795 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7796 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7797 Assert(puIntrState);
7798 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7799
7800 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7801 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7802
7803#ifdef VBOX_STRICT
7804 /* Validate the error-code-valid bit for hardware exceptions. */
7805 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7806 {
7807 switch (uVector)
7808 {
7809 case X86_XCPT_PF:
7810 case X86_XCPT_DF:
7811 case X86_XCPT_TS:
7812 case X86_XCPT_NP:
7813 case X86_XCPT_SS:
7814 case X86_XCPT_GP:
7815 case X86_XCPT_AC:
7816 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7817 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7818 /* fallthru */
7819 default:
7820 break;
7821 }
7822 }
7823#endif
7824
7825 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7826 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7827 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7828
7829 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7830
7831 /* We require CR0 to check if the guest is in real-mode. */
7832 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7833 AssertRCReturn(rc, rc);
7834
7835 /*
7836 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7837 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7838 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7839 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7840 */
7841 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7842 {
7843 PVM pVM = pVCpu->CTX_SUFF(pVM);
7844 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7845 {
7846 Assert(PDMVmmDevHeapIsEnabled(pVM));
7847 Assert(pVM->hm.s.vmx.pRealModeTSS);
7848
7849 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7850 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7851 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7852 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7853 AssertRCReturn(rc, rc);
7854 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7855
7856 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7857 size_t const cbIdtEntry = sizeof(X86IDTR16);
7858 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7859 {
7860 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7861 if (uVector == X86_XCPT_DF)
7862 return VINF_EM_RESET;
7863 else if (uVector == X86_XCPT_GP)
7864 {
7865 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7866 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7867 }
7868
7869 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7870 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7871 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7872 }
7873
7874 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7875 uint16_t uGuestIp = pMixedCtx->ip;
7876 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7877 {
7878 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7879 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7880 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7881 }
7882 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7883 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7884
7885 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7886 X86IDTR16 IdtEntry;
7887 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7888 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7889 AssertRCReturn(rc, rc);
7890
7891 /* Construct the stack frame for the interrupt/exception handler. */
7892 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7893 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7894 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7895 AssertRCReturn(rc, rc);
7896
7897 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7898 if (rc == VINF_SUCCESS)
7899 {
7900 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7901 pMixedCtx->rip = IdtEntry.offSel;
7902 pMixedCtx->cs.Sel = IdtEntry.uSel;
7903 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7904 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7905 && uVector == X86_XCPT_PF)
7906 {
7907 pMixedCtx->cr2 = GCPtrFaultAddress;
7908 }
7909
7910 /* If any other guest-state bits are changed here, make sure to update
7911 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7912 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7913 | HM_CHANGED_GUEST_RIP
7914 | HM_CHANGED_GUEST_RFLAGS
7915 | HM_CHANGED_GUEST_RSP);
7916
7917 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7918 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7919 {
7920 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7921 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7922 Log4(("Clearing inhibition due to STI.\n"));
7923 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7924 }
7925 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7926
7927 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7928 it, if we are returning to ring-3 before executing guest code. */
7929 pVCpu->hm.s.Event.fPending = false;
7930 }
7931 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7932 return rc;
7933 }
7934 else
7935 {
7936 /*
7937 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7938 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7939 */
7940 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7941 }
7942 }
7943
7944 /* Validate. */
7945 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7946 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7947 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7948
7949 /* Inject. */
7950 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7951 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7952 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7953 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7954
7955 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7956 && uVector == X86_XCPT_PF)
7957 {
7958 pMixedCtx->cr2 = GCPtrFaultAddress;
7959 }
7960
7961 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7962 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7963
7964 AssertRCReturn(rc, rc);
7965 return rc;
7966}
7967
7968
7969/**
7970 * Clears the interrupt-window exiting control in the VMCS and if necessary
7971 * clears the current event in the VMCS as well.
7972 *
7973 * @returns VBox status code.
7974 * @param pVCpu Pointer to the VMCPU.
7975 *
7976 * @remarks Use this function only to clear events that have not yet been
7977 * delivered to the guest but are injected in the VMCS!
7978 * @remarks No-long-jump zone!!!
7979 */
7980static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7981{
7982 int rc;
7983 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7984
7985 /* Clear interrupt-window exiting control. */
7986 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7987 {
7988 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7989 Assert(!pVCpu->hm.s.Event.fPending);
7990 }
7991
7992 /* Clear NMI-window exiting control. */
7993 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7994 {
7995 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7996 Assert(!pVCpu->hm.s.Event.fPending);
7997 }
7998
7999 if (!pVCpu->hm.s.Event.fPending)
8000 return;
8001
8002#ifdef VBOX_STRICT
8003 uint32_t u32EntryInfo;
8004 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8005 AssertRC(rc);
8006 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8007#endif
8008
8009 /* Clear the entry-interruption field (including the valid bit). */
8010 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8011 AssertRC(rc);
8012
8013 /* Clear the pending debug exception field. */
8014 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8015 AssertRC(rc);
8016
8017 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8018 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8019}
8020
8021
8022/**
8023 * Enters the VT-x session.
8024 *
8025 * @returns VBox status code.
8026 * @param pVM Pointer to the VM.
8027 * @param pVCpu Pointer to the VMCPU.
8028 * @param pCpu Pointer to the CPU info struct.
8029 */
8030VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8031{
8032 AssertPtr(pVM);
8033 AssertPtr(pVCpu);
8034 Assert(pVM->hm.s.vmx.fSupported);
8035 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8036 NOREF(pCpu); NOREF(pVM);
8037
8038 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8039 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8040
8041#ifdef VBOX_STRICT
8042 /* Make sure we're in VMX root mode. */
8043 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8044 if (!(u32HostCR4 & X86_CR4_VMXE))
8045 {
8046 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8047 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8048 }
8049#endif
8050
8051 /*
8052 * Load the VCPU's VMCS as the current (and active) one.
8053 */
8054 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8055 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8056 if (RT_FAILURE(rc))
8057 return rc;
8058
8059 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8060 pVCpu->hm.s.fLeaveDone = false;
8061 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8062
8063 return VINF_SUCCESS;
8064}
8065
8066
8067/**
8068 * The thread-context callback (only on platforms which support it).
8069 *
8070 * @param enmEvent The thread-context event.
8071 * @param pVCpu Pointer to the VMCPU.
8072 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8073 * @thread EMT(pVCpu)
8074 */
8075VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8076{
8077 NOREF(fGlobalInit);
8078
8079 switch (enmEvent)
8080 {
8081 case RTTHREADCTXEVENT_PREEMPTING:
8082 {
8083 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8084 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8085 VMCPU_ASSERT_EMT(pVCpu);
8086
8087 PVM pVM = pVCpu->CTX_SUFF(pVM);
8088 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8089
8090 /* No longjmps (logger flushes, locks) in this fragile context. */
8091 VMMRZCallRing3Disable(pVCpu);
8092 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8093
8094 /*
8095 * Restore host-state (FPU, debug etc.)
8096 */
8097 if (!pVCpu->hm.s.fLeaveDone)
8098 {
8099 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8100 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8101 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8102 pVCpu->hm.s.fLeaveDone = true;
8103 }
8104
8105 /* Leave HM context, takes care of local init (term). */
8106 int rc = HMR0LeaveCpu(pVCpu);
8107 AssertRC(rc); NOREF(rc);
8108
8109 /* Restore longjmp state. */
8110 VMMRZCallRing3Enable(pVCpu);
8111 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8112 break;
8113 }
8114
8115 case RTTHREADCTXEVENT_RESUMED:
8116 {
8117 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8118 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8119 VMCPU_ASSERT_EMT(pVCpu);
8120
8121 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8122 VMMRZCallRing3Disable(pVCpu);
8123 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8124
8125 /* Initialize the bare minimum state required for HM. This takes care of
8126 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8127 int rc = HMR0EnterCpu(pVCpu);
8128 AssertRC(rc);
8129 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8130
8131 /* Load the active VMCS as the current one. */
8132 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8133 {
8134 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8135 AssertRC(rc); NOREF(rc);
8136 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8137 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8138 }
8139 pVCpu->hm.s.fLeaveDone = false;
8140
8141 /* Restore longjmp state. */
8142 VMMRZCallRing3Enable(pVCpu);
8143 break;
8144 }
8145
8146 default:
8147 break;
8148 }
8149}
8150
8151
8152/**
8153 * Saves the host state in the VMCS host-state.
8154 * Sets up the VM-exit MSR-load area.
8155 *
8156 * The CPU state will be loaded from these fields on every successful VM-exit.
8157 *
8158 * @returns VBox status code.
8159 * @param pVM Pointer to the VM.
8160 * @param pVCpu Pointer to the VMCPU.
8161 *
8162 * @remarks No-long-jump zone!!!
8163 */
8164static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8165{
8166 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8167
8168 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8169 return VINF_SUCCESS;
8170
8171 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8172 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8173
8174 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8175 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8176
8177 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8178 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8179
8180 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8181 return rc;
8182}
8183
8184
8185/**
8186 * Saves the host state in the VMCS host-state.
8187 *
8188 * @returns VBox status code.
8189 * @param pVM Pointer to the VM.
8190 * @param pVCpu Pointer to the VMCPU.
8191 *
8192 * @remarks No-long-jump zone!!!
8193 */
8194VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8195{
8196 AssertPtr(pVM);
8197 AssertPtr(pVCpu);
8198
8199 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8200
8201 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8202 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8203 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8204 return hmR0VmxSaveHostState(pVM, pVCpu);
8205}
8206
8207
8208/**
8209 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8210 * loaded from these fields on every successful VM-entry.
8211 *
8212 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8213 * Sets up the VM-entry controls.
8214 * Sets up the appropriate VMX non-root function to execute guest code based on
8215 * the guest CPU mode.
8216 *
8217 * @returns VBox status code.
8218 * @param pVM Pointer to the VM.
8219 * @param pVCpu Pointer to the VMCPU.
8220 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8221 * out-of-sync. Make sure to update the required fields
8222 * before using them.
8223 *
8224 * @remarks No-long-jump zone!!!
8225 */
8226static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8227{
8228 AssertPtr(pVM);
8229 AssertPtr(pVCpu);
8230 AssertPtr(pMixedCtx);
8231 HMVMX_ASSERT_PREEMPT_SAFE();
8232
8233#ifdef LOG_ENABLED
8234 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8235 * probably not initialized yet? Anyway this will do for now.
8236 *
8237 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8238 * interface and disable ring-3 calls when thread-context hooks are not
8239 * available. */
8240 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8241 VMMR0LogFlushDisable(pVCpu);
8242#endif
8243
8244 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8245
8246 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8247
8248 /* Determine real-on-v86 mode. */
8249 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8250 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8251 && CPUMIsGuestInRealModeEx(pMixedCtx))
8252 {
8253 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8254 }
8255
8256 /*
8257 * Load the guest-state into the VMCS.
8258 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8259 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8260 */
8261 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8262 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8263
8264 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8265 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8266 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8267
8268 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8269 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8270 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8271
8272 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8273 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8274
8275 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8276 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8277
8278 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8279 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8280 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8281
8282 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8283 determine we don't have to swap EFER after all. */
8284 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8285 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8286
8287 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8288 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8289
8290 /*
8291 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8292 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8293 */
8294 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8295 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8296
8297 /* Clear any unused and reserved bits. */
8298 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8299
8300#ifdef LOG_ENABLED
8301 /* Only reenable log-flushing if the caller has it enabled. */
8302 if (!fCallerDisabledLogFlush)
8303 VMMR0LogFlushEnable(pVCpu);
8304#endif
8305
8306 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8307 return rc;
8308}
8309
8310
8311/**
8312 * Loads the state shared between the host and guest into the VMCS.
8313 *
8314 * @param pVM Pointer to the VM.
8315 * @param pVCpu Pointer to the VMCPU.
8316 * @param pCtx Pointer to the guest-CPU context.
8317 *
8318 * @remarks No-long-jump zone!!!
8319 */
8320static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8321{
8322 NOREF(pVM);
8323
8324 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8325 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8326
8327 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8328 {
8329 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8330 AssertRC(rc);
8331 }
8332
8333 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8334 {
8335 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8336 AssertRC(rc);
8337
8338 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8339 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8340 {
8341 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8342 AssertRC(rc);
8343 }
8344 }
8345
8346 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8347 {
8348#if HC_ARCH_BITS == 64
8349 if (pVM->hm.s.fAllow64BitGuests)
8350 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8351#endif
8352 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8353 }
8354
8355 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8356 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8357}
8358
8359
8360/**
8361 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8362 *
8363 * @param pVM Pointer to the VM.
8364 * @param pVCpu Pointer to the VMCPU.
8365 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8366 * out-of-sync. Make sure to update the required fields
8367 * before using them.
8368 */
8369DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8370{
8371 HMVMX_ASSERT_PREEMPT_SAFE();
8372
8373 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8374#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8375 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8376#endif
8377
8378 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8379 {
8380 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8381 AssertRC(rc);
8382 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8383 }
8384 else if (HMCPU_CF_VALUE(pVCpu))
8385 {
8386 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8387 AssertRC(rc);
8388 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8389 }
8390
8391 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8392 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8393 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8394 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8395}
8396
8397
8398/**
8399 * Does the preparations before executing guest code in VT-x.
8400 *
8401 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8402 * recompiler. We must be cautious what we do here regarding committing
8403 * guest-state information into the VMCS assuming we assuredly execute the
8404 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8405 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8406 * so that the recompiler can (and should) use them when it resumes guest
8407 * execution. Otherwise such operations must be done when we can no longer
8408 * exit to ring-3.
8409 *
8410 * @returns Strict VBox status code.
8411 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8412 * have been disabled.
8413 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8414 * double-fault into the guest.
8415 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8416 *
8417 * @param pVM Pointer to the VM.
8418 * @param pVCpu Pointer to the VMCPU.
8419 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8420 * out-of-sync. Make sure to update the required fields
8421 * before using them.
8422 * @param pVmxTransient Pointer to the VMX transient structure.
8423 */
8424static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8425{
8426 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8427
8428#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8429 PGMRZDynMapFlushAutoSet(pVCpu);
8430#endif
8431
8432 /* Check force flag actions that might require us to go back to ring-3. */
8433 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8434 if (rc != VINF_SUCCESS)
8435 return rc;
8436
8437#ifndef IEM_VERIFICATION_MODE_FULL
8438 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8439 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8440 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8441 {
8442 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8443 RTGCPHYS GCPhysApicBase;
8444 GCPhysApicBase = pMixedCtx->msrApicBase;
8445 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8446
8447 /* Unalias any existing mapping. */
8448 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8449 AssertRCReturn(rc, rc);
8450
8451 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8452 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8453 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8454 AssertRCReturn(rc, rc);
8455
8456 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8457 }
8458#endif /* !IEM_VERIFICATION_MODE_FULL */
8459
8460 if (TRPMHasTrap(pVCpu))
8461 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8462 else if (!pVCpu->hm.s.Event.fPending)
8463 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8464
8465 /*
8466 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8467 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8468 */
8469 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8470 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8471 {
8472 Assert(rc == VINF_EM_RESET);
8473 return rc;
8474 }
8475
8476 /*
8477 * Load the guest state bits, we can handle longjmps/getting preempted here.
8478 *
8479 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8480 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8481 * Hence, this needs to be done -after- injection of events.
8482 */
8483 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8484
8485 /*
8486 * No longjmps to ring-3 from this point on!!!
8487 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8488 * This also disables flushing of the R0-logger instance (if any).
8489 */
8490 VMMRZCallRing3Disable(pVCpu);
8491
8492 /*
8493 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8494 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8495 *
8496 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8497 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8498 *
8499 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8500 * executing guest code.
8501 */
8502 pVmxTransient->uEflags = ASMIntDisableFlags();
8503 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8504 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8505 {
8506 hmR0VmxClearEventVmcs(pVCpu);
8507 ASMSetFlags(pVmxTransient->uEflags);
8508 VMMRZCallRing3Enable(pVCpu);
8509 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8510 return VINF_EM_RAW_TO_R3;
8511 }
8512
8513 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8514 {
8515 hmR0VmxClearEventVmcs(pVCpu);
8516 ASMSetFlags(pVmxTransient->uEflags);
8517 VMMRZCallRing3Enable(pVCpu);
8518 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8519 return VINF_EM_RAW_INTERRUPT;
8520 }
8521
8522 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8523 pVCpu->hm.s.Event.fPending = false;
8524
8525 return VINF_SUCCESS;
8526}
8527
8528
8529/**
8530 * Prepares to run guest code in VT-x and we've committed to doing so. This
8531 * means there is no backing out to ring-3 or anywhere else at this
8532 * point.
8533 *
8534 * @param pVM Pointer to the VM.
8535 * @param pVCpu Pointer to the VMCPU.
8536 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8537 * out-of-sync. Make sure to update the required fields
8538 * before using them.
8539 * @param pVmxTransient Pointer to the VMX transient structure.
8540 *
8541 * @remarks Called with preemption disabled.
8542 * @remarks No-long-jump zone!!!
8543 */
8544static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8545{
8546 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8547 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8548 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8549
8550 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8551 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8552
8553#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8554 if (!CPUMIsGuestFPUStateActive(pVCpu))
8555 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8556 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8557#endif
8558
8559 if ( pVCpu->hm.s.fUseGuestFpu
8560 && !CPUMIsGuestFPUStateActive(pVCpu))
8561 {
8562 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8563 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8564 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8565 }
8566
8567 /*
8568 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8569 */
8570 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8571 && pVCpu->hm.s.vmx.cMsrs > 0)
8572 {
8573 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8574 }
8575
8576 /*
8577 * Load the host state bits as we may've been preempted (only happens when
8578 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8579 */
8580 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8581 {
8582 /* This ASSUMES that pfnStartVM has been set up already. */
8583 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8584 AssertRC(rc);
8585 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8586 }
8587 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8588
8589 /*
8590 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8591 */
8592 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8593 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8594 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8595
8596 /* Store status of the shared guest-host state at the time of VM-entry. */
8597#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8598 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8599 {
8600 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8601 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8602 }
8603 else
8604#endif
8605 {
8606 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8607 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8608 }
8609 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8610
8611 /*
8612 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8613 */
8614 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8615 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8616
8617 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8618 RTCPUID idCurrentCpu = pCpu->idCpu;
8619 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8620 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8621 {
8622 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8623 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8624 }
8625
8626 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8627 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8628 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8629 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8630
8631 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8632
8633 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8634 to start executing. */
8635
8636 /*
8637 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8638 */
8639 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8640 {
8641 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8642 {
8643 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8644 AssertRC(rc2);
8645 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8646 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8647 true /* fUpdateHostMsr */);
8648 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8649 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8650 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8651 }
8652 else
8653 {
8654 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8655 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8656 }
8657 }
8658
8659#ifdef VBOX_STRICT
8660 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8661 hmR0VmxCheckHostEferMsr(pVCpu);
8662 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8663#endif
8664#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8665 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8666 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8667 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8668#endif
8669}
8670
8671
8672/**
8673 * Performs some essential restoration of state after running guest code in
8674 * VT-x.
8675 *
8676 * @param pVM Pointer to the VM.
8677 * @param pVCpu Pointer to the VMCPU.
8678 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8679 * out-of-sync. Make sure to update the required fields
8680 * before using them.
8681 * @param pVmxTransient Pointer to the VMX transient structure.
8682 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8683 *
8684 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8685 *
8686 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8687 * unconditionally when it is safe to do so.
8688 */
8689static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8690{
8691 NOREF(pVM);
8692
8693 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8694
8695 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8696 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8697 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8698 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8699 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8700
8701 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8702 {
8703 /** @todo Find a way to fix hardcoding a guestimate. */
8704 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8705 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8706 }
8707
8708 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8709 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8710 Assert(!(ASMGetFlags() & X86_EFL_IF));
8711 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8712
8713#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8714 if (CPUMIsGuestFPUStateActive(pVCpu))
8715 {
8716 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8717 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8718 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8719 }
8720#endif
8721
8722#if HC_ARCH_BITS == 64
8723 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8724#endif
8725 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8726#ifdef VBOX_STRICT
8727 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8728#endif
8729 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8730 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8731
8732 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8733 uint32_t uExitReason;
8734 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8735 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8736 AssertRC(rc);
8737 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8738 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8739
8740 /* Update the VM-exit history array. */
8741 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8742
8743 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8744 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8745 {
8746 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8747 pVmxTransient->fVMEntryFailed));
8748 return;
8749 }
8750
8751 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8752 {
8753 /** @todo We can optimize this by only syncing with our force-flags when
8754 * really needed and keeping the VMCS state as it is for most
8755 * VM-exits. */
8756 /* Update the guest interruptibility-state from the VMCS. */
8757 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8758
8759#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8760 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8761 AssertRC(rc);
8762#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8763 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8764 AssertRC(rc);
8765#endif
8766
8767 /*
8768 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8769 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8770 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8771 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8772 */
8773 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8774 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8775 {
8776 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8777 AssertRC(rc);
8778 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8779 }
8780 }
8781}
8782
8783
8784/**
8785 * Runs the guest code using VT-x the normal way.
8786 *
8787 * @returns VBox status code.
8788 * @param pVM Pointer to the VM.
8789 * @param pVCpu Pointer to the VMCPU.
8790 * @param pCtx Pointer to the guest-CPU context.
8791 *
8792 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8793 */
8794static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8795{
8796 VMXTRANSIENT VmxTransient;
8797 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8798 int rc = VERR_INTERNAL_ERROR_5;
8799 uint32_t cLoops = 0;
8800
8801 for (;; cLoops++)
8802 {
8803 Assert(!HMR0SuspendPending());
8804 HMVMX_ASSERT_CPU_SAFE();
8805
8806 /* Preparatory work for running guest code, this may force us to return
8807 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8808 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8809 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8810 if (rc != VINF_SUCCESS)
8811 break;
8812
8813 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8814 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8815 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8816
8817 /* Restore any residual host-state and save any bits shared between host
8818 and guest into the guest-CPU state. Re-enables interrupts! */
8819 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8820
8821 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8822 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8823 {
8824 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8825 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8826 return rc;
8827 }
8828
8829 /* Handle the VM-exit. */
8830 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8831 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8832 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8833 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8834 HMVMX_START_EXIT_DISPATCH_PROF();
8835#ifdef HMVMX_USE_FUNCTION_TABLE
8836 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8837#else
8838 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8839#endif
8840 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8841 if (rc != VINF_SUCCESS)
8842 break;
8843 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8844 {
8845 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8846 rc = VINF_EM_RAW_INTERRUPT;
8847 break;
8848 }
8849 }
8850
8851 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8852 return rc;
8853}
8854
8855
8856/**
8857 * Single steps guest code using VT-x.
8858 *
8859 * @returns VBox status code.
8860 * @param pVM Pointer to the VM.
8861 * @param pVCpu Pointer to the VMCPU.
8862 * @param pCtx Pointer to the guest-CPU context.
8863 *
8864 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8865 */
8866static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8867{
8868 VMXTRANSIENT VmxTransient;
8869 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8870 int rc = VERR_INTERNAL_ERROR_5;
8871 uint32_t cLoops = 0;
8872 uint16_t uCsStart = pCtx->cs.Sel;
8873 uint64_t uRipStart = pCtx->rip;
8874
8875 for (;; cLoops++)
8876 {
8877 Assert(!HMR0SuspendPending());
8878 HMVMX_ASSERT_CPU_SAFE();
8879
8880 /* Preparatory work for running guest code, this may force us to return
8881 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8882 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8883 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8884 if (rc != VINF_SUCCESS)
8885 break;
8886
8887 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8888 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8889 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8890
8891 /* Restore any residual host-state and save any bits shared between host
8892 and guest into the guest-CPU state. Re-enables interrupts! */
8893 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8894
8895 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8896 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8897 {
8898 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8899 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8900 return rc;
8901 }
8902
8903 /* Handle the VM-exit. */
8904 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8905 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8906 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8907 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8908 HMVMX_START_EXIT_DISPATCH_PROF();
8909#ifdef HMVMX_USE_FUNCTION_TABLE
8910 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8911#else
8912 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8913#endif
8914 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8915 if (rc != VINF_SUCCESS)
8916 break;
8917 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8918 {
8919 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8920 rc = VINF_EM_RAW_INTERRUPT;
8921 break;
8922 }
8923
8924 /*
8925 * Did the RIP change, if so, consider it a single step.
8926 * Otherwise, make sure one of the TFs gets set.
8927 */
8928 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8929 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8930 AssertRCReturn(rc2, rc2);
8931 if ( pCtx->rip != uRipStart
8932 || pCtx->cs.Sel != uCsStart)
8933 {
8934 rc = VINF_EM_DBG_STEPPED;
8935 break;
8936 }
8937 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8938 }
8939
8940 /*
8941 * Clear the X86_EFL_TF if necessary.
8942 */
8943 if (pVCpu->hm.s.fClearTrapFlag)
8944 {
8945 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8946 AssertRCReturn(rc2, rc2);
8947 pVCpu->hm.s.fClearTrapFlag = false;
8948 pCtx->eflags.Bits.u1TF = 0;
8949 }
8950 /** @todo there seems to be issues with the resume flag when the monitor trap
8951 * flag is pending without being used. Seen early in bios init when
8952 * accessing APIC page in protected mode. */
8953
8954 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8955 return rc;
8956}
8957
8958
8959/**
8960 * Runs the guest code using VT-x.
8961 *
8962 * @returns VBox status code.
8963 * @param pVM Pointer to the VM.
8964 * @param pVCpu Pointer to the VMCPU.
8965 * @param pCtx Pointer to the guest-CPU context.
8966 */
8967VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8968{
8969 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8970 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8971 HMVMX_ASSERT_PREEMPT_SAFE();
8972
8973 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8974
8975 int rc;
8976 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8977 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8978 else
8979 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8980
8981 if (rc == VERR_EM_INTERPRETER)
8982 rc = VINF_EM_RAW_EMULATE_INSTR;
8983 else if (rc == VINF_EM_RESET)
8984 rc = VINF_EM_TRIPLE_FAULT;
8985
8986 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8987 if (RT_FAILURE(rc2))
8988 {
8989 pVCpu->hm.s.u32HMError = rc;
8990 rc = rc2;
8991 }
8992 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8993 return rc;
8994}
8995
8996
8997#ifndef HMVMX_USE_FUNCTION_TABLE
8998DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8999{
9000#ifdef DEBUG_ramshankar
9001# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9002# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9003#endif
9004 int rc;
9005 switch (rcReason)
9006 {
9007 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9008 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9009 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9010 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9011 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9012 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9013 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9014 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9015 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9016 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9017 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9018 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9019 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9020 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9021 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9022 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9023 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9024 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9025 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9026 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9027 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9028 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9029 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041
9042 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9043 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9044 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9045 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9046 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9047 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9048 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9049 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9050 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9051
9052 case VMX_EXIT_VMCLEAR:
9053 case VMX_EXIT_VMLAUNCH:
9054 case VMX_EXIT_VMPTRLD:
9055 case VMX_EXIT_VMPTRST:
9056 case VMX_EXIT_VMREAD:
9057 case VMX_EXIT_VMRESUME:
9058 case VMX_EXIT_VMWRITE:
9059 case VMX_EXIT_VMXOFF:
9060 case VMX_EXIT_VMXON:
9061 case VMX_EXIT_INVEPT:
9062 case VMX_EXIT_INVVPID:
9063 case VMX_EXIT_VMFUNC:
9064 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9065 break;
9066 default:
9067 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9068 break;
9069 }
9070 return rc;
9071}
9072#endif
9073
9074#ifdef DEBUG
9075/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9076# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9077 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9078
9079# define HMVMX_ASSERT_PREEMPT_CPUID() \
9080 do \
9081 { \
9082 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9083 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9084 } while (0)
9085
9086# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9087 do { \
9088 AssertPtr(pVCpu); \
9089 AssertPtr(pMixedCtx); \
9090 AssertPtr(pVmxTransient); \
9091 Assert(pVmxTransient->fVMEntryFailed == false); \
9092 Assert(ASMIntAreEnabled()); \
9093 HMVMX_ASSERT_PREEMPT_SAFE(); \
9094 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9095 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9096 HMVMX_ASSERT_PREEMPT_SAFE(); \
9097 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9098 HMVMX_ASSERT_PREEMPT_CPUID(); \
9099 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9100 } while (0)
9101
9102# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9103 do { \
9104 Log4Func(("\n")); \
9105 } while (0)
9106#else /* Release builds */
9107# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9108 do { \
9109 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9110 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9111 } while (0)
9112# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9113#endif
9114
9115
9116/**
9117 * Advances the guest RIP after reading it from the VMCS.
9118 *
9119 * @returns VBox status code.
9120 * @param pVCpu Pointer to the VMCPU.
9121 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9122 * out-of-sync. Make sure to update the required fields
9123 * before using them.
9124 * @param pVmxTransient Pointer to the VMX transient structure.
9125 *
9126 * @remarks No-long-jump zone!!!
9127 */
9128DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9129{
9130 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9131 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9132 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9133 AssertRCReturn(rc, rc);
9134
9135 pMixedCtx->rip += pVmxTransient->cbInstr;
9136 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9137
9138 /*
9139 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9140 * pending debug exception field as it takes care of priority of events.
9141 *
9142 * See Intel spec. 32.2.1 "Debug Exceptions".
9143 */
9144 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9145
9146 return rc;
9147}
9148
9149
9150/**
9151 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9152 * and update error record fields accordingly.
9153 *
9154 * @return VMX_IGS_* return codes.
9155 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9156 * wrong with the guest state.
9157 *
9158 * @param pVM Pointer to the VM.
9159 * @param pVCpu Pointer to the VMCPU.
9160 * @param pCtx Pointer to the guest-CPU state.
9161 *
9162 * @remarks This function assumes our cache of the VMCS controls
9163 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9164 */
9165static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9166{
9167#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9168#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9169 uError = (err); \
9170 break; \
9171 } else do { } while (0)
9172
9173 int rc;
9174 uint32_t uError = VMX_IGS_ERROR;
9175 uint32_t u32Val;
9176 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9177
9178 do
9179 {
9180 /*
9181 * CR0.
9182 */
9183 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9184 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9185 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9186 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9187 if (fUnrestrictedGuest)
9188 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9189
9190 uint32_t u32GuestCR0;
9191 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9192 AssertRCBreak(rc);
9193 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9194 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9195 if ( !fUnrestrictedGuest
9196 && (u32GuestCR0 & X86_CR0_PG)
9197 && !(u32GuestCR0 & X86_CR0_PE))
9198 {
9199 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9200 }
9201
9202 /*
9203 * CR4.
9204 */
9205 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9206 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9207
9208 uint32_t u32GuestCR4;
9209 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9210 AssertRCBreak(rc);
9211 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9212 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9213
9214 /*
9215 * IA32_DEBUGCTL MSR.
9216 */
9217 uint64_t u64Val;
9218 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9219 AssertRCBreak(rc);
9220 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9221 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9222 {
9223 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9224 }
9225 uint64_t u64DebugCtlMsr = u64Val;
9226
9227#ifdef VBOX_STRICT
9228 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9229 AssertRCBreak(rc);
9230 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9231#endif
9232 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9233
9234 /*
9235 * RIP and RFLAGS.
9236 */
9237 uint32_t u32Eflags;
9238#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9239 if (HMVMX_IS_64BIT_HOST_MODE())
9240 {
9241 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9242 AssertRCBreak(rc);
9243 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9244 if ( !fLongModeGuest
9245 || !pCtx->cs.Attr.n.u1Long)
9246 {
9247 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9248 }
9249 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9250 * must be identical if the "IA-32e mode guest" VM-entry
9251 * control is 1 and CS.L is 1. No check applies if the
9252 * CPU supports 64 linear-address bits. */
9253
9254 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9255 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9256 AssertRCBreak(rc);
9257 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9258 VMX_IGS_RFLAGS_RESERVED);
9259 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9260 u32Eflags = u64Val;
9261 }
9262 else
9263#endif
9264 {
9265 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9266 AssertRCBreak(rc);
9267 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9268 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9269 }
9270
9271 if ( fLongModeGuest
9272 || ( fUnrestrictedGuest
9273 && !(u32GuestCR0 & X86_CR0_PE)))
9274 {
9275 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9276 }
9277
9278 uint32_t u32EntryInfo;
9279 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9280 AssertRCBreak(rc);
9281 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9282 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9283 {
9284 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9285 }
9286
9287 /*
9288 * 64-bit checks.
9289 */
9290#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9291 if (HMVMX_IS_64BIT_HOST_MODE())
9292 {
9293 if ( fLongModeGuest
9294 && !fUnrestrictedGuest)
9295 {
9296 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9297 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9298 }
9299
9300 if ( !fLongModeGuest
9301 && (u32GuestCR4 & X86_CR4_PCIDE))
9302 {
9303 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9304 }
9305
9306 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9307 * 51:32 beyond the processor's physical-address width are 0. */
9308
9309 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9310 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9311 {
9312 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9313 }
9314
9315 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9316 AssertRCBreak(rc);
9317 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9318
9319 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9320 AssertRCBreak(rc);
9321 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9322 }
9323#endif
9324
9325 /*
9326 * PERF_GLOBAL MSR.
9327 */
9328 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9329 {
9330 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9331 AssertRCBreak(rc);
9332 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9333 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9334 }
9335
9336 /*
9337 * PAT MSR.
9338 */
9339 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9340 {
9341 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9342 AssertRCBreak(rc);
9343 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9344 for (unsigned i = 0; i < 8; i++)
9345 {
9346 uint8_t u8Val = (u64Val & 0x7);
9347 if ( u8Val != 0 /* UC */
9348 || u8Val != 1 /* WC */
9349 || u8Val != 4 /* WT */
9350 || u8Val != 5 /* WP */
9351 || u8Val != 6 /* WB */
9352 || u8Val != 7 /* UC- */)
9353 {
9354 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9355 }
9356 u64Val >>= 3;
9357 }
9358 }
9359
9360 /*
9361 * EFER MSR.
9362 */
9363 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9364 {
9365 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9366 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9367 AssertRCBreak(rc);
9368 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9369 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9370 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9371 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9372 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9373 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9374 VMX_IGS_EFER_LMA_PG_MISMATCH);
9375 }
9376
9377 /*
9378 * Segment registers.
9379 */
9380 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9381 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9382 if (!(u32Eflags & X86_EFL_VM))
9383 {
9384 /* CS */
9385 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9386 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9387 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9388 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9389 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9390 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9391 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9392 /* CS cannot be loaded with NULL in protected mode. */
9393 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9394 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9395 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9396 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9397 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9398 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9399 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9400 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9401 else
9402 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9403
9404 /* SS */
9405 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9406 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9407 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9408 if ( !(pCtx->cr0 & X86_CR0_PE)
9409 || pCtx->cs.Attr.n.u4Type == 3)
9410 {
9411 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9412 }
9413 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9414 {
9415 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9416 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9417 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9418 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9419 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9420 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9421 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9422 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9423 }
9424
9425 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9426 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9427 {
9428 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9429 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9430 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9431 || pCtx->ds.Attr.n.u4Type > 11
9432 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9433 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9434 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9435 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9436 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9437 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9438 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9439 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9440 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9441 }
9442 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9443 {
9444 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9445 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9446 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9447 || pCtx->es.Attr.n.u4Type > 11
9448 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9449 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9450 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9451 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9452 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9453 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9454 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9455 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9456 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9457 }
9458 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9459 {
9460 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9461 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9462 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9463 || pCtx->fs.Attr.n.u4Type > 11
9464 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9465 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9466 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9467 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9468 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9469 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9470 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9471 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9472 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9473 }
9474 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9475 {
9476 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9477 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9478 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9479 || pCtx->gs.Attr.n.u4Type > 11
9480 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9481 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9482 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9483 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9484 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9485 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9486 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9487 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9488 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9489 }
9490 /* 64-bit capable CPUs. */
9491#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9492 if (HMVMX_IS_64BIT_HOST_MODE())
9493 {
9494 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9495 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9496 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9497 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9498 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9499 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9500 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9501 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9502 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9503 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9504 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9505 }
9506#endif
9507 }
9508 else
9509 {
9510 /* V86 mode checks. */
9511 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9512 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9513 {
9514 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9515 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9516 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9517 }
9518 else
9519 {
9520 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9521 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9522 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9523 }
9524
9525 /* CS */
9526 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9527 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9528 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9529 /* SS */
9530 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9531 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9532 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9533 /* DS */
9534 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9535 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9536 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9537 /* ES */
9538 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9539 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9540 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9541 /* FS */
9542 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9543 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9544 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9545 /* GS */
9546 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9547 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9548 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9549 /* 64-bit capable CPUs. */
9550#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9551 if (HMVMX_IS_64BIT_HOST_MODE())
9552 {
9553 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9554 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9555 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9556 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9557 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9558 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9559 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9560 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9561 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9562 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9563 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9564 }
9565#endif
9566 }
9567
9568 /*
9569 * TR.
9570 */
9571 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9572 /* 64-bit capable CPUs. */
9573#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9574 if (HMVMX_IS_64BIT_HOST_MODE())
9575 {
9576 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9577 }
9578#endif
9579 if (fLongModeGuest)
9580 {
9581 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9582 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9583 }
9584 else
9585 {
9586 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9587 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9588 VMX_IGS_TR_ATTR_TYPE_INVALID);
9589 }
9590 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9591 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9592 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9593 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9594 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9595 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9596 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9597 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9598
9599 /*
9600 * GDTR and IDTR.
9601 */
9602#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9603 if (HMVMX_IS_64BIT_HOST_MODE())
9604 {
9605 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9606 AssertRCBreak(rc);
9607 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9608
9609 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9610 AssertRCBreak(rc);
9611 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9612 }
9613#endif
9614
9615 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9616 AssertRCBreak(rc);
9617 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9618
9619 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9620 AssertRCBreak(rc);
9621 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9622
9623 /*
9624 * Guest Non-Register State.
9625 */
9626 /* Activity State. */
9627 uint32_t u32ActivityState;
9628 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9629 AssertRCBreak(rc);
9630 HMVMX_CHECK_BREAK( !u32ActivityState
9631 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9632 VMX_IGS_ACTIVITY_STATE_INVALID);
9633 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9634 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9635 uint32_t u32IntrState;
9636 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9637 AssertRCBreak(rc);
9638 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9639 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9640 {
9641 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9642 }
9643
9644 /** @todo Activity state and injecting interrupts. Left as a todo since we
9645 * currently don't use activity states but ACTIVE. */
9646
9647 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9648 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9649
9650 /* Guest interruptibility-state. */
9651 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9652 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9653 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9654 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9655 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9656 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9657 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9658 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9659 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9660 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9661 {
9662 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9663 {
9664 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9665 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9666 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9667 }
9668 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9669 {
9670 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9671 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9672 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9673 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9674 }
9675 }
9676 /** @todo Assumes the processor is not in SMM. */
9677 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9678 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9679 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9680 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9681 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9682 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9683 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9684 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9685 {
9686 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9687 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9688 }
9689
9690 /* Pending debug exceptions. */
9691 if (HMVMX_IS_64BIT_HOST_MODE())
9692 {
9693 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9694 AssertRCBreak(rc);
9695 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9696 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9697 u32Val = u64Val; /* For pending debug exceptions checks below. */
9698 }
9699 else
9700 {
9701 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9702 AssertRCBreak(rc);
9703 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9704 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9705 }
9706
9707 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9708 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9709 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9710 {
9711 if ( (u32Eflags & X86_EFL_TF)
9712 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9713 {
9714 /* Bit 14 is PendingDebug.BS. */
9715 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9716 }
9717 if ( !(u32Eflags & X86_EFL_TF)
9718 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9719 {
9720 /* Bit 14 is PendingDebug.BS. */
9721 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9722 }
9723 }
9724
9725 /* VMCS link pointer. */
9726 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9727 AssertRCBreak(rc);
9728 if (u64Val != UINT64_C(0xffffffffffffffff))
9729 {
9730 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9731 /** @todo Bits beyond the processor's physical-address width MBZ. */
9732 /** @todo 32-bit located in memory referenced by value of this field (as a
9733 * physical address) must contain the processor's VMCS revision ID. */
9734 /** @todo SMM checks. */
9735 }
9736
9737 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9738 * not using Nested Paging? */
9739 if ( pVM->hm.s.fNestedPaging
9740 && !fLongModeGuest
9741 && CPUMIsGuestInPAEModeEx(pCtx))
9742 {
9743 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9744 AssertRCBreak(rc);
9745 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9746
9747 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9748 AssertRCBreak(rc);
9749 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9750
9751 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9752 AssertRCBreak(rc);
9753 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9754
9755 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9756 AssertRCBreak(rc);
9757 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9758 }
9759
9760 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9761 if (uError == VMX_IGS_ERROR)
9762 uError = VMX_IGS_REASON_NOT_FOUND;
9763 } while (0);
9764
9765 pVCpu->hm.s.u32HMError = uError;
9766 return uError;
9767
9768#undef HMVMX_ERROR_BREAK
9769#undef HMVMX_CHECK_BREAK
9770}
9771
9772/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9773/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9774/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9775
9776/** @name VM-exit handlers.
9777 * @{
9778 */
9779
9780/**
9781 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9782 */
9783HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9784{
9785 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9786 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9787 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9788 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9789 return VINF_SUCCESS;
9790 return VINF_EM_RAW_INTERRUPT;
9791}
9792
9793
9794/**
9795 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9796 */
9797HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9798{
9799 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9800 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9801
9802 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9803 AssertRCReturn(rc, rc);
9804
9805 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9806 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9807 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9808 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9809
9810 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9811 {
9812 /*
9813 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9814 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9815 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9816 *
9817 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9818 */
9819 VMXDispatchHostNmi();
9820 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9821 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9822 return VINF_SUCCESS;
9823 }
9824
9825 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9826 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9827 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9828 {
9829 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9830 return VINF_SUCCESS;
9831 }
9832 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9833 {
9834 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9835 return rc;
9836 }
9837
9838 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9839 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9840 switch (uIntType)
9841 {
9842 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9843 Assert(uVector == X86_XCPT_DB);
9844 /* no break */
9845 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9846 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9847 /* no break */
9848 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9849 {
9850 switch (uVector)
9851 {
9852 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9853 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9854 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9855 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9856 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9857 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9858#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9859 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9860 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9861 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9862 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9863 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9864 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9865 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9866 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9867 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9868 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9869 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9870 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9871#endif
9872 default:
9873 {
9874 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9875 AssertRCReturn(rc, rc);
9876
9877 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9878 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9879 {
9880 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9881 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9882 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9883
9884 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9885 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9886 AssertRCReturn(rc, rc);
9887 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9888 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9889 0 /* GCPtrFaultAddress */);
9890 AssertRCReturn(rc, rc);
9891 }
9892 else
9893 {
9894 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9895 pVCpu->hm.s.u32HMError = uVector;
9896 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9897 }
9898 break;
9899 }
9900 }
9901 break;
9902 }
9903
9904 default:
9905 {
9906 pVCpu->hm.s.u32HMError = uExitIntInfo;
9907 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9908 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9909 break;
9910 }
9911 }
9912 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9913 return rc;
9914}
9915
9916
9917/**
9918 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9919 */
9920HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9921{
9922 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9923
9924 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9925 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9926
9927 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9928 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9929 return VINF_SUCCESS;
9930}
9931
9932
9933/**
9934 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9935 */
9936HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9937{
9938 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9939 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
9940 {
9941 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9942 HMVMX_RETURN_UNEXPECTED_EXIT();
9943 }
9944
9945 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
9946
9947 /*
9948 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
9949 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
9950 */
9951 uint32_t uIntrState = 0;
9952 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9953 AssertRCReturn(rc, rc);
9954
9955 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
9956 if ( fBlockSti
9957 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
9958 {
9959 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9960 }
9961
9962 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
9963 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
9964
9965 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9966 return VINF_SUCCESS;
9967}
9968
9969
9970/**
9971 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9972 */
9973HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9974{
9975 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9976 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9977 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9978}
9979
9980
9981/**
9982 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9983 */
9984HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9985{
9986 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9987 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9988 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9989}
9990
9991
9992/**
9993 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9994 */
9995HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9996{
9997 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9998 PVM pVM = pVCpu->CTX_SUFF(pVM);
9999 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10000 if (RT_LIKELY(rc == VINF_SUCCESS))
10001 {
10002 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10003 Assert(pVmxTransient->cbInstr == 2);
10004 }
10005 else
10006 {
10007 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10008 rc = VERR_EM_INTERPRETER;
10009 }
10010 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10011 return rc;
10012}
10013
10014
10015/**
10016 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10017 */
10018HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10019{
10020 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10021 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10022 AssertRCReturn(rc, rc);
10023
10024 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10025 return VINF_EM_RAW_EMULATE_INSTR;
10026
10027 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10028 HMVMX_RETURN_UNEXPECTED_EXIT();
10029}
10030
10031
10032/**
10033 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10034 */
10035HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10036{
10037 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10038 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10039 AssertRCReturn(rc, rc);
10040
10041 PVM pVM = pVCpu->CTX_SUFF(pVM);
10042 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10043 if (RT_LIKELY(rc == VINF_SUCCESS))
10044 {
10045 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10046 Assert(pVmxTransient->cbInstr == 2);
10047 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10048 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10049 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10050 }
10051 else
10052 {
10053 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
10054 rc = VERR_EM_INTERPRETER;
10055 }
10056 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10057 return rc;
10058}
10059
10060
10061/**
10062 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10063 */
10064HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10065{
10066 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10067 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10068 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10069 AssertRCReturn(rc, rc);
10070
10071 PVM pVM = pVCpu->CTX_SUFF(pVM);
10072 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10073 if (RT_LIKELY(rc == VINF_SUCCESS))
10074 {
10075 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10076 Assert(pVmxTransient->cbInstr == 3);
10077 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10078 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10079 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10080 }
10081 else
10082 {
10083 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10084 rc = VERR_EM_INTERPRETER;
10085 }
10086 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10087 return rc;
10088}
10089
10090
10091/**
10092 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10093 */
10094HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10095{
10096 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10097 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10098 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10099 AssertRCReturn(rc, rc);
10100
10101 PVM pVM = pVCpu->CTX_SUFF(pVM);
10102 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10103 if (RT_LIKELY(rc == VINF_SUCCESS))
10104 {
10105 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10106 Assert(pVmxTransient->cbInstr == 2);
10107 }
10108 else
10109 {
10110 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10111 rc = VERR_EM_INTERPRETER;
10112 }
10113 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10114 return rc;
10115}
10116
10117
10118/**
10119 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10120 */
10121HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10122{
10123 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10124
10125 int rc = VERR_NOT_SUPPORTED;
10126 if (GIMAreHypercallsEnabled(pVCpu))
10127 {
10128 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10129 AssertRCReturn(rc, rc);
10130
10131 rc = GIMHypercall(pVCpu, pMixedCtx);
10132 }
10133 if (rc != VINF_SUCCESS)
10134 {
10135 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10136 rc = VINF_SUCCESS;
10137 }
10138
10139 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10140 return rc;
10141}
10142
10143
10144/**
10145 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10146 */
10147HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10148{
10149 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10150 PVM pVM = pVCpu->CTX_SUFF(pVM);
10151 Assert(!pVM->hm.s.fNestedPaging);
10152
10153 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10154 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10155 AssertRCReturn(rc, rc);
10156
10157 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10158 rc = VBOXSTRICTRC_VAL(rc2);
10159 if (RT_LIKELY(rc == VINF_SUCCESS))
10160 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10161 else
10162 {
10163 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10164 pVmxTransient->uExitQualification, rc));
10165 }
10166 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10167 return rc;
10168}
10169
10170
10171/**
10172 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10173 */
10174HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10175{
10176 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10177 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10178 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10179 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10180 AssertRCReturn(rc, rc);
10181
10182 PVM pVM = pVCpu->CTX_SUFF(pVM);
10183 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10184 if (RT_LIKELY(rc == VINF_SUCCESS))
10185 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10186 else
10187 {
10188 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10189 rc = VERR_EM_INTERPRETER;
10190 }
10191 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10192 return rc;
10193}
10194
10195
10196/**
10197 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10198 */
10199HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10200{
10201 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10202 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10203 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10204 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10205 AssertRCReturn(rc, rc);
10206
10207 PVM pVM = pVCpu->CTX_SUFF(pVM);
10208 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10209 rc = VBOXSTRICTRC_VAL(rc2);
10210 if (RT_LIKELY( rc == VINF_SUCCESS
10211 || rc == VINF_EM_HALT))
10212 {
10213 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10214 AssertRCReturn(rc3, rc3);
10215
10216 if ( rc == VINF_EM_HALT
10217 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10218 {
10219 rc = VINF_SUCCESS;
10220 }
10221 }
10222 else
10223 {
10224 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10225 rc = VERR_EM_INTERPRETER;
10226 }
10227 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10228 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10229 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10230 return rc;
10231}
10232
10233
10234/**
10235 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10236 */
10237HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10238{
10239 /*
10240 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10241 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10242 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10243 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10244 */
10245 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10246 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10247 HMVMX_RETURN_UNEXPECTED_EXIT();
10248}
10249
10250
10251/**
10252 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10253 */
10254HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10255{
10256 /*
10257 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10258 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10259 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10260 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10261 */
10262 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10263 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10264 HMVMX_RETURN_UNEXPECTED_EXIT();
10265}
10266
10267
10268/**
10269 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10270 */
10271HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10272{
10273 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10274 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10275 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10276 HMVMX_RETURN_UNEXPECTED_EXIT();
10277}
10278
10279
10280/**
10281 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10282 */
10283HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10284{
10285 /*
10286 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10287 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10288 * See Intel spec. 25.3 "Other Causes of VM-exits".
10289 */
10290 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10291 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10292 HMVMX_RETURN_UNEXPECTED_EXIT();
10293}
10294
10295
10296/**
10297 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10298 * VM-exit.
10299 */
10300HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10301{
10302 /*
10303 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10304 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10305 *
10306 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10307 * See Intel spec. "23.8 Restrictions on VMX operation".
10308 */
10309 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10310 return VINF_SUCCESS;
10311}
10312
10313
10314/**
10315 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10316 * VM-exit.
10317 */
10318HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10319{
10320 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10321 return VINF_EM_RESET;
10322}
10323
10324
10325/**
10326 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10327 */
10328HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10329{
10330 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10331 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10332 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10333 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10334 AssertRCReturn(rc, rc);
10335
10336 pMixedCtx->rip++;
10337 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10338 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10339 rc = VINF_SUCCESS;
10340 else
10341 rc = VINF_EM_HALT;
10342
10343 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10344 return rc;
10345}
10346
10347
10348/**
10349 * VM-exit handler for instructions that result in a #UD exception delivered to
10350 * the guest.
10351 */
10352HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10353{
10354 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10355 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10356 return VINF_SUCCESS;
10357}
10358
10359
10360/**
10361 * VM-exit handler for expiry of the VMX preemption timer.
10362 */
10363HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10364{
10365 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10366
10367 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10368 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10369
10370 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10371 PVM pVM = pVCpu->CTX_SUFF(pVM);
10372 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10373 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10374 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10375}
10376
10377
10378/**
10379 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10380 */
10381HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10382{
10383 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10384
10385 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10386 /** @todo check if XSETBV is supported by the recompiler. */
10387 return VERR_EM_INTERPRETER;
10388}
10389
10390
10391/**
10392 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10393 */
10394HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10395{
10396 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10397
10398 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10399 /** @todo implement EMInterpretInvpcid() */
10400 return VERR_EM_INTERPRETER;
10401}
10402
10403
10404/**
10405 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10406 * Error VM-exit.
10407 */
10408HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10409{
10410 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10411 AssertRCReturn(rc, rc);
10412
10413 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10414 AssertRCReturn(rc, rc);
10415
10416 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10417 NOREF(uInvalidReason);
10418
10419#ifdef VBOX_STRICT
10420 uint32_t uIntrState;
10421 HMVMXHCUINTREG uHCReg;
10422 uint64_t u64Val;
10423 uint32_t u32Val;
10424
10425 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10426 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10427 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10428 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10429 AssertRCReturn(rc, rc);
10430
10431 Log4(("uInvalidReason %u\n", uInvalidReason));
10432 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10433 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10434 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10435 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10436
10437 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10438 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10439 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10440 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10441 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10442 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10443 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10444 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10445 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10446 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10447 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10448 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10449#else
10450 NOREF(pVmxTransient);
10451#endif
10452
10453 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10454 return VERR_VMX_INVALID_GUEST_STATE;
10455}
10456
10457
10458/**
10459 * VM-exit handler for VM-entry failure due to an MSR-load
10460 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10461 */
10462HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10463{
10464 NOREF(pVmxTransient);
10465 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10466 HMVMX_RETURN_UNEXPECTED_EXIT();
10467}
10468
10469
10470/**
10471 * VM-exit handler for VM-entry failure due to a machine-check event
10472 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10473 */
10474HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10475{
10476 NOREF(pVmxTransient);
10477 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10478 HMVMX_RETURN_UNEXPECTED_EXIT();
10479}
10480
10481
10482/**
10483 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10484 * theory.
10485 */
10486HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10487{
10488 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10489 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10490 return VERR_VMX_UNDEFINED_EXIT_CODE;
10491}
10492
10493
10494/**
10495 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10496 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10497 * Conditional VM-exit.
10498 */
10499HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10500{
10501 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10502
10503 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10504 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10505 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10506 return VERR_EM_INTERPRETER;
10507 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10508 HMVMX_RETURN_UNEXPECTED_EXIT();
10509}
10510
10511
10512/**
10513 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10514 */
10515HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10516{
10517 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10518
10519 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10521 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10522 return VERR_EM_INTERPRETER;
10523 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10524 HMVMX_RETURN_UNEXPECTED_EXIT();
10525}
10526
10527
10528/**
10529 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10530 */
10531HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10532{
10533 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10534
10535 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10536 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10537 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10538 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10539 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10540 {
10541 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10542 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10543 }
10544 AssertRCReturn(rc, rc);
10545 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10546
10547#ifdef VBOX_STRICT
10548 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10549 {
10550 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10551 && pMixedCtx->ecx != MSR_K6_EFER)
10552 {
10553 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10554 HMVMX_RETURN_UNEXPECTED_EXIT();
10555 }
10556# if HC_ARCH_BITS == 64
10557 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10558 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10559 {
10560 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10561 HMVMX_RETURN_UNEXPECTED_EXIT();
10562 }
10563# endif
10564 }
10565#endif
10566
10567 PVM pVM = pVCpu->CTX_SUFF(pVM);
10568 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10569 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10570 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10571 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10572 if (RT_LIKELY(rc == VINF_SUCCESS))
10573 {
10574 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10575 Assert(pVmxTransient->cbInstr == 2);
10576 }
10577 return rc;
10578}
10579
10580
10581/**
10582 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10583 */
10584HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10585{
10586 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10587 PVM pVM = pVCpu->CTX_SUFF(pVM);
10588 int rc = VINF_SUCCESS;
10589
10590 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10591 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10592 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10593 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10594 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10595 {
10596 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10597 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10598 }
10599 AssertRCReturn(rc, rc);
10600 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10601
10602 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10603 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10604 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10605
10606 if (RT_LIKELY(rc == VINF_SUCCESS))
10607 {
10608 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10609
10610 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10611 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10612 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10613 {
10614 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10615 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10616 EMInterpretWrmsr() changes it. */
10617 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10618 }
10619 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10620 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10621 else if (pMixedCtx->ecx == MSR_K6_EFER)
10622 {
10623 /*
10624 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10625 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10626 * the other bits as well, SCE and NXE. See @bugref{7368}.
10627 */
10628 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10629 }
10630
10631 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10632 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10633 {
10634 switch (pMixedCtx->ecx)
10635 {
10636 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10637 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10638 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10639 case MSR_K8_FS_BASE: /* no break */
10640 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10641 case MSR_K6_EFER: /* already handled above */ break;
10642 default:
10643 {
10644 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10645 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10646#if HC_ARCH_BITS == 64
10647 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10648 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10649#endif
10650 break;
10651 }
10652 }
10653 }
10654#ifdef VBOX_STRICT
10655 else
10656 {
10657 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10658 switch (pMixedCtx->ecx)
10659 {
10660 case MSR_IA32_SYSENTER_CS:
10661 case MSR_IA32_SYSENTER_EIP:
10662 case MSR_IA32_SYSENTER_ESP:
10663 case MSR_K8_FS_BASE:
10664 case MSR_K8_GS_BASE:
10665 {
10666 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10667 HMVMX_RETURN_UNEXPECTED_EXIT();
10668 }
10669
10670 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10671 default:
10672 {
10673 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10674 {
10675 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10676 if (pMixedCtx->ecx != MSR_K6_EFER)
10677 {
10678 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10679 pMixedCtx->ecx));
10680 HMVMX_RETURN_UNEXPECTED_EXIT();
10681 }
10682 }
10683
10684#if HC_ARCH_BITS == 64
10685 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10686 {
10687 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10688 HMVMX_RETURN_UNEXPECTED_EXIT();
10689 }
10690#endif
10691 break;
10692 }
10693 }
10694 }
10695#endif /* VBOX_STRICT */
10696 }
10697 return rc;
10698}
10699
10700
10701/**
10702 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10703 */
10704HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10705{
10706 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10707
10708 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10709 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10710 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10711 return VERR_EM_INTERPRETER;
10712 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10713 HMVMX_RETURN_UNEXPECTED_EXIT();
10714}
10715
10716
10717/**
10718 * VM-exit handler for when the TPR value is lowered below the specified
10719 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10720 */
10721HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10722{
10723 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10724 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10725
10726 /*
10727 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10728 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10729 * resume guest execution.
10730 */
10731 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10732 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10733 return VINF_SUCCESS;
10734}
10735
10736
10737/**
10738 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10739 * VM-exit.
10740 *
10741 * @retval VINF_SUCCESS when guest execution can continue.
10742 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10743 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10744 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10745 * recompiler.
10746 */
10747HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10748{
10749 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10750 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10751 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10752 AssertRCReturn(rc, rc);
10753
10754 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10755 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10756 PVM pVM = pVCpu->CTX_SUFF(pVM);
10757 switch (uAccessType)
10758 {
10759 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10760 {
10761#if 0
10762 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10763 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10764#else
10765 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10766 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10767 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10768#endif
10769 AssertRCReturn(rc, rc);
10770
10771 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10772 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10773 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10774 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10775
10776 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10777 {
10778 case 0: /* CR0 */
10779 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10780 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10781 break;
10782 case 2: /* CR2 */
10783 /* Nothing to do here, CR2 it's not part of the VMCS. */
10784 break;
10785 case 3: /* CR3 */
10786 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10787 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10788 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10789 break;
10790 case 4: /* CR4 */
10791 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10792 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10793 break;
10794 case 8: /* CR8 */
10795 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10796 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10797 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10798 break;
10799 default:
10800 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10801 break;
10802 }
10803
10804 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10805 break;
10806 }
10807
10808 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10809 {
10810 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10811 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10812 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10813 AssertRCReturn(rc, rc);
10814 Assert( !pVM->hm.s.fNestedPaging
10815 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10816 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10817
10818 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10819 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10820 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10821
10822 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10823 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10824 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10825 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10827 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10828 break;
10829 }
10830
10831 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10832 {
10833 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10834 AssertRCReturn(rc, rc);
10835 rc = EMInterpretCLTS(pVM, pVCpu);
10836 AssertRCReturn(rc, rc);
10837 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10838 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10839 Log4(("CRX CLTS write rc=%d\n", rc));
10840 break;
10841 }
10842
10843 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10844 {
10845 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10846 AssertRCReturn(rc, rc);
10847 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10848 if (RT_LIKELY(rc == VINF_SUCCESS))
10849 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10850 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10851 Log4(("CRX LMSW write rc=%d\n", rc));
10852 break;
10853 }
10854
10855 default:
10856 {
10857 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10858 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10859 }
10860 }
10861
10862 /* Validate possible error codes. */
10863 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10864 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10865 if (RT_SUCCESS(rc))
10866 {
10867 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10868 AssertRCReturn(rc2, rc2);
10869 }
10870
10871 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10872 return rc;
10873}
10874
10875
10876/**
10877 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10878 * VM-exit.
10879 */
10880HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10881{
10882 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10883 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10884
10885 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10886 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10887 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10888 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10889 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10890 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10891 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10892 AssertRCReturn(rc2, rc2);
10893
10894 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10895 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10896 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10897 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10898 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10899 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10900 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10901 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10902
10903 /* I/O operation lookup arrays. */
10904 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10905 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10906
10907 VBOXSTRICTRC rcStrict;
10908 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10909 uint32_t const cbInstr = pVmxTransient->cbInstr;
10910 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10911 PVM pVM = pVCpu->CTX_SUFF(pVM);
10912 if (fIOString)
10913 {
10914#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10915 /*
10916 * INS/OUTS - I/O String instruction.
10917 *
10918 * Use instruction-information if available, otherwise fall back on
10919 * interpreting the instruction.
10920 */
10921 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10922 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10923 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10924 {
10925 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10926 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10927 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10928 AssertRCReturn(rc2, rc2);
10929 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10930 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10931 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10932 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10933 if (fIOWrite)
10934 {
10935 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10936 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10937 }
10938 else
10939 {
10940 /*
10941 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10942 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10943 * See Intel Instruction spec. for "INS".
10944 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10945 */
10946 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10947 }
10948 }
10949 else
10950 {
10951 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10952 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10953 AssertRCReturn(rc2, rc2);
10954 rcStrict = IEMExecOne(pVCpu);
10955 }
10956 /** @todo IEM needs to be setting these flags somehow. */
10957 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10958 fUpdateRipAlready = true;
10959#else
10960 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10961 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10962 if (RT_SUCCESS(rcStrict))
10963 {
10964 if (fIOWrite)
10965 {
10966 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10967 (DISCPUMODE)pDis->uAddrMode, cbValue);
10968 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10969 }
10970 else
10971 {
10972 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10973 (DISCPUMODE)pDis->uAddrMode, cbValue);
10974 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10975 }
10976 }
10977 else
10978 {
10979 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10980 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10981 }
10982#endif
10983 }
10984 else
10985 {
10986 /*
10987 * IN/OUT - I/O instruction.
10988 */
10989 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10990 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
10991 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10992 if (fIOWrite)
10993 {
10994 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10995 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10996 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10997 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10998 }
10999 else
11000 {
11001 uint32_t u32Result = 0;
11002 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11003 if (IOM_SUCCESS(rcStrict))
11004 {
11005 /* Save result of I/O IN instr. in AL/AX/EAX. */
11006 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11007 }
11008 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11009 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11010 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11011 }
11012 }
11013
11014 if (IOM_SUCCESS(rcStrict))
11015 {
11016 if (!fUpdateRipAlready)
11017 {
11018 pMixedCtx->rip += cbInstr;
11019 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11020 }
11021
11022 /*
11023 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11024 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11025 */
11026 if (fIOString)
11027 {
11028 /** @todo Single-step for INS/OUTS with REP prefix? */
11029 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11030 }
11031 else if (fStepping)
11032 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11033
11034 /*
11035 * If any I/O breakpoints are armed, we need to check if one triggered
11036 * and take appropriate action.
11037 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11038 */
11039 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11040 AssertRCReturn(rc2, rc2);
11041
11042 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11043 * execution engines about whether hyper BPs and such are pending. */
11044 uint32_t const uDr7 = pMixedCtx->dr[7];
11045 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11046 && X86_DR7_ANY_RW_IO(uDr7)
11047 && (pMixedCtx->cr4 & X86_CR4_DE))
11048 || DBGFBpIsHwIoArmed(pVM)))
11049 {
11050 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11051
11052 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11053 VMMRZCallRing3Disable(pVCpu);
11054 HM_DISABLE_PREEMPT_IF_NEEDED();
11055
11056 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
11057
11058 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11059 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11060 {
11061 /* Raise #DB. */
11062 if (fIsGuestDbgActive)
11063 ASMSetDR6(pMixedCtx->dr[6]);
11064 if (pMixedCtx->dr[7] != uDr7)
11065 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11066
11067 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11068 }
11069 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11070 else if ( rcStrict2 != VINF_SUCCESS
11071 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11072 rcStrict = rcStrict2;
11073
11074 HM_RESTORE_PREEMPT_IF_NEEDED();
11075 VMMRZCallRing3Enable(pVCpu);
11076 }
11077 }
11078
11079#ifdef DEBUG
11080 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11081 Assert(!fIOWrite);
11082 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11083 Assert(fIOWrite);
11084 else
11085 {
11086 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11087 * statuses, that the VMM device and some others may return. See
11088 * IOM_SUCCESS() for guidance. */
11089 AssertMsg( RT_FAILURE(rcStrict)
11090 || rcStrict == VINF_SUCCESS
11091 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11092 || rcStrict == VINF_EM_DBG_BREAKPOINT
11093 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11094 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11095 }
11096#endif
11097
11098 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11099 return VBOXSTRICTRC_TODO(rcStrict);
11100}
11101
11102
11103/**
11104 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11105 * VM-exit.
11106 */
11107HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11108{
11109 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11110
11111 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11112 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11113 AssertRCReturn(rc, rc);
11114 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11115 {
11116 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11117 AssertRCReturn(rc, rc);
11118 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11119 {
11120 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11121
11122 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11123 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11124
11125 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11126 Assert(!pVCpu->hm.s.Event.fPending);
11127 pVCpu->hm.s.Event.fPending = true;
11128 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11129 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11130 AssertRCReturn(rc, rc);
11131 if (fErrorCodeValid)
11132 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11133 else
11134 pVCpu->hm.s.Event.u32ErrCode = 0;
11135 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11136 && uVector == X86_XCPT_PF)
11137 {
11138 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11139 }
11140
11141 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11142 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11143 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11144 }
11145 }
11146
11147 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11148 * emulation. */
11149 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11150 return VERR_EM_INTERPRETER;
11151}
11152
11153
11154/**
11155 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11156 */
11157HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11158{
11159 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11160 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11161 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11162 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11163 AssertRCReturn(rc, rc);
11164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11165 return VINF_EM_DBG_STEPPED;
11166}
11167
11168
11169/**
11170 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11171 */
11172HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11173{
11174 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11175
11176 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11177 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11178 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11179 return VINF_SUCCESS;
11180 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11181 return rc;
11182
11183#if 0
11184 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11185 * just sync the whole thing. */
11186 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11187#else
11188 /* Aggressive state sync. for now. */
11189 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11190 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11191 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11192#endif
11193 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11194 AssertRCReturn(rc, rc);
11195
11196 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11197 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11198 switch (uAccessType)
11199 {
11200 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11201 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11202 {
11203 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11204 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
11205 {
11206 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11207 }
11208
11209 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11210 GCPhys &= PAGE_BASE_GC_MASK;
11211 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11212 PVM pVM = pVCpu->CTX_SUFF(pVM);
11213 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11214 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11215
11216 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11217 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11218 CPUMCTX2CORE(pMixedCtx), GCPhys);
11219 rc = VBOXSTRICTRC_VAL(rc2);
11220 Log4(("ApicAccess rc=%d\n", rc));
11221 if ( rc == VINF_SUCCESS
11222 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11223 || rc == VERR_PAGE_NOT_PRESENT)
11224 {
11225 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11226 | HM_CHANGED_GUEST_RSP
11227 | HM_CHANGED_GUEST_RFLAGS
11228 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11229 rc = VINF_SUCCESS;
11230 }
11231 break;
11232 }
11233
11234 default:
11235 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11236 rc = VINF_EM_RAW_EMULATE_INSTR;
11237 break;
11238 }
11239
11240 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11241 return rc;
11242}
11243
11244
11245/**
11246 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11247 * VM-exit.
11248 */
11249HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11250{
11251 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11252
11253 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11254 if (pVmxTransient->fWasGuestDebugStateActive)
11255 {
11256 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11257 HMVMX_RETURN_UNEXPECTED_EXIT();
11258 }
11259
11260 int rc = VERR_INTERNAL_ERROR_5;
11261 if ( !DBGFIsStepping(pVCpu)
11262 && !pVCpu->hm.s.fSingleInstruction
11263 && !pVmxTransient->fWasHyperDebugStateActive)
11264 {
11265 /* Don't intercept MOV DRx and #DB any more. */
11266 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11267 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11268 AssertRCReturn(rc, rc);
11269
11270 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11271 {
11272#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11273 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11274 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11275 AssertRCReturn(rc, rc);
11276#endif
11277 }
11278
11279 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11280 VMMRZCallRing3Disable(pVCpu);
11281 HM_DISABLE_PREEMPT_IF_NEEDED();
11282
11283 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11284 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11285 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11286
11287 HM_RESTORE_PREEMPT_IF_NEEDED();
11288 VMMRZCallRing3Enable(pVCpu);
11289
11290#ifdef VBOX_WITH_STATISTICS
11291 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11292 AssertRCReturn(rc, rc);
11293 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11295 else
11296 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11297#endif
11298 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11299 return VINF_SUCCESS;
11300 }
11301
11302 /*
11303 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11304 * Update the segment registers and DR7 from the CPU.
11305 */
11306 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11307 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11308 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11309 AssertRCReturn(rc, rc);
11310 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11311
11312 PVM pVM = pVCpu->CTX_SUFF(pVM);
11313 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11314 {
11315 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11316 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11317 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11318 if (RT_SUCCESS(rc))
11319 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11320 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11321 }
11322 else
11323 {
11324 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11325 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11326 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11327 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11328 }
11329
11330 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11331 if (RT_SUCCESS(rc))
11332 {
11333 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11334 AssertRCReturn(rc2, rc2);
11335 }
11336 return rc;
11337}
11338
11339
11340/**
11341 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11342 * Conditional VM-exit.
11343 */
11344HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11345{
11346 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11347 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11348
11349 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11350 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11351 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11352 return VINF_SUCCESS;
11353 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11354 return rc;
11355
11356 RTGCPHYS GCPhys = 0;
11357 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11358
11359#if 0
11360 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11361#else
11362 /* Aggressive state sync. for now. */
11363 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11364 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11365 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11366#endif
11367 AssertRCReturn(rc, rc);
11368
11369 /*
11370 * If we succeed, resume guest execution.
11371 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11372 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11373 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11374 * weird case. See @bugref{6043}.
11375 */
11376 PVM pVM = pVCpu->CTX_SUFF(pVM);
11377 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11378 rc = VBOXSTRICTRC_VAL(rc2);
11379 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11380 if ( rc == VINF_SUCCESS
11381 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11382 || rc == VERR_PAGE_NOT_PRESENT)
11383 {
11384 /* Successfully handled MMIO operation. */
11385 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11386 | HM_CHANGED_GUEST_RSP
11387 | HM_CHANGED_GUEST_RFLAGS
11388 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11389 rc = VINF_SUCCESS;
11390 }
11391 return rc;
11392}
11393
11394
11395/**
11396 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11397 * VM-exit.
11398 */
11399HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11400{
11401 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11402 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11403
11404 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11405 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11406 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11407 return VINF_SUCCESS;
11408 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11409 return rc;
11410
11411 RTGCPHYS GCPhys = 0;
11412 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11413 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11414#if 0
11415 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11416#else
11417 /* Aggressive state sync. for now. */
11418 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11419 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11420 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11421#endif
11422 AssertRCReturn(rc, rc);
11423
11424 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11425 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11426
11427 RTGCUINT uErrorCode = 0;
11428 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11429 uErrorCode |= X86_TRAP_PF_ID;
11430 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11431 uErrorCode |= X86_TRAP_PF_RW;
11432 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11433 uErrorCode |= X86_TRAP_PF_P;
11434
11435 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11436
11437 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11438 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11439
11440 /* Handle the pagefault trap for the nested shadow table. */
11441 PVM pVM = pVCpu->CTX_SUFF(pVM);
11442 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11443 TRPMResetTrap(pVCpu);
11444
11445 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11446 if ( rc == VINF_SUCCESS
11447 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11448 || rc == VERR_PAGE_NOT_PRESENT)
11449 {
11450 /* Successfully synced our nested page tables. */
11451 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11452 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11453 | HM_CHANGED_GUEST_RSP
11454 | HM_CHANGED_GUEST_RFLAGS);
11455 return VINF_SUCCESS;
11456 }
11457
11458 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11459 return rc;
11460}
11461
11462/** @} */
11463
11464/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11465/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11466/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11467
11468/** @name VM-exit exception handlers.
11469 * @{
11470 */
11471
11472/**
11473 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11474 */
11475static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11476{
11477 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11478 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11479
11480 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11481 AssertRCReturn(rc, rc);
11482
11483 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11484 {
11485 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11486 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11487
11488 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11489 * provides VM-exit instruction length. If this causes problem later,
11490 * disassemble the instruction like it's done on AMD-V. */
11491 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11492 AssertRCReturn(rc2, rc2);
11493 return rc;
11494 }
11495
11496 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11497 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11498 return rc;
11499}
11500
11501
11502/**
11503 * VM-exit exception handler for #BP (Breakpoint exception).
11504 */
11505static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11506{
11507 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11508 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11509
11510 /** @todo Try optimize this by not saving the entire guest state unless
11511 * really needed. */
11512 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11513 AssertRCReturn(rc, rc);
11514
11515 PVM pVM = pVCpu->CTX_SUFF(pVM);
11516 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11517 if (rc == VINF_EM_RAW_GUEST_TRAP)
11518 {
11519 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11520 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11521 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11522 AssertRCReturn(rc, rc);
11523
11524 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11525 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11526 }
11527
11528 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11529 return rc;
11530}
11531
11532
11533/**
11534 * VM-exit exception handler for #DB (Debug exception).
11535 */
11536static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11537{
11538 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11539 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11540 Log6(("XcptDB\n"));
11541
11542 /*
11543 * Get the DR6-like values from the exit qualification and pass it to DBGF
11544 * for processing.
11545 */
11546 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11547 AssertRCReturn(rc, rc);
11548
11549 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11550 uint64_t uDR6 = X86_DR6_INIT_VAL;
11551 uDR6 |= ( pVmxTransient->uExitQualification
11552 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11553
11554 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11555 if (rc == VINF_EM_RAW_GUEST_TRAP)
11556 {
11557 /*
11558 * The exception was for the guest. Update DR6, DR7.GD and
11559 * IA32_DEBUGCTL.LBR before forwarding it.
11560 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11561 */
11562 VMMRZCallRing3Disable(pVCpu);
11563 HM_DISABLE_PREEMPT_IF_NEEDED();
11564
11565 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11566 pMixedCtx->dr[6] |= uDR6;
11567 if (CPUMIsGuestDebugStateActive(pVCpu))
11568 ASMSetDR6(pMixedCtx->dr[6]);
11569
11570 HM_RESTORE_PREEMPT_IF_NEEDED();
11571 VMMRZCallRing3Enable(pVCpu);
11572
11573 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11574 AssertRCReturn(rc, rc);
11575
11576 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11577 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11578
11579 /* Paranoia. */
11580 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11581 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11582
11583 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11584 AssertRCReturn(rc, rc);
11585
11586 /*
11587 * Raise #DB in the guest.
11588 *
11589 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11590 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11591 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11592 *
11593 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11594 */
11595 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11596 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11597 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11598 AssertRCReturn(rc, rc);
11599 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11600 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11601 return VINF_SUCCESS;
11602 }
11603
11604 /*
11605 * Not a guest trap, must be a hypervisor related debug event then.
11606 * Update DR6 in case someone is interested in it.
11607 */
11608 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11609 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11610 CPUMSetHyperDR6(pVCpu, uDR6);
11611
11612 return rc;
11613}
11614
11615
11616/**
11617 * VM-exit exception handler for #NM (Device-not-available exception: floating
11618 * point exception).
11619 */
11620static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11621{
11622 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11623
11624 /* We require CR0 and EFER. EFER is always up-to-date. */
11625 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11626 AssertRCReturn(rc, rc);
11627
11628 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11629 VMMRZCallRing3Disable(pVCpu);
11630 HM_DISABLE_PREEMPT_IF_NEEDED();
11631
11632 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11633 if (pVmxTransient->fWasGuestFPUStateActive)
11634 {
11635 rc = VINF_EM_RAW_GUEST_TRAP;
11636 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11637 }
11638 else
11639 {
11640#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11641 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11642#endif
11643 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11644 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11645 }
11646
11647 HM_RESTORE_PREEMPT_IF_NEEDED();
11648 VMMRZCallRing3Enable(pVCpu);
11649
11650 if (rc == VINF_SUCCESS)
11651 {
11652 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11653 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11654 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11655 pVCpu->hm.s.fUseGuestFpu = true;
11656 }
11657 else
11658 {
11659 /* Forward #NM to the guest. */
11660 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11661 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11662 AssertRCReturn(rc, rc);
11663 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11664 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11665 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11666 }
11667
11668 return VINF_SUCCESS;
11669}
11670
11671
11672/**
11673 * VM-exit exception handler for #GP (General-protection exception).
11674 *
11675 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11676 */
11677static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11678{
11679 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11680 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11681
11682 int rc = VERR_INTERNAL_ERROR_5;
11683 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11684 {
11685#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11686 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11687 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11688 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11689 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11690 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11691 AssertRCReturn(rc, rc);
11692 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11693 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11694 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11695 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11696 return rc;
11697#else
11698 /* We don't intercept #GP. */
11699 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11700 NOREF(pVmxTransient);
11701 return VERR_VMX_UNEXPECTED_EXCEPTION;
11702#endif
11703 }
11704
11705 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11706 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11707
11708 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11709 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11710 AssertRCReturn(rc, rc);
11711
11712 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11713 uint32_t cbOp = 0;
11714 PVM pVM = pVCpu->CTX_SUFF(pVM);
11715 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11716 if (RT_SUCCESS(rc))
11717 {
11718 rc = VINF_SUCCESS;
11719 Assert(cbOp == pDis->cbInstr);
11720 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11721 switch (pDis->pCurInstr->uOpcode)
11722 {
11723 case OP_CLI:
11724 {
11725 pMixedCtx->eflags.Bits.u1IF = 0;
11726 pMixedCtx->rip += pDis->cbInstr;
11727 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11728 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11729 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11730 break;
11731 }
11732
11733 case OP_STI:
11734 {
11735 pMixedCtx->eflags.Bits.u1IF = 1;
11736 pMixedCtx->rip += pDis->cbInstr;
11737 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11738 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11739 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11740 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11741 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11742 break;
11743 }
11744
11745 case OP_HLT:
11746 {
11747 rc = VINF_EM_HALT;
11748 pMixedCtx->rip += pDis->cbInstr;
11749 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11750 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11751 break;
11752 }
11753
11754 case OP_POPF:
11755 {
11756 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11757 uint32_t cbParm;
11758 uint32_t uMask;
11759 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11760 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11761 {
11762 cbParm = 4;
11763 uMask = 0xffffffff;
11764 }
11765 else
11766 {
11767 cbParm = 2;
11768 uMask = 0xffff;
11769 }
11770
11771 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11772 RTGCPTR GCPtrStack = 0;
11773 X86EFLAGS Eflags;
11774 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11775 &GCPtrStack);
11776 if (RT_SUCCESS(rc))
11777 {
11778 Assert(sizeof(Eflags.u32) >= cbParm);
11779 Eflags.u32 = 0;
11780 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11781 }
11782 if (RT_FAILURE(rc))
11783 {
11784 rc = VERR_EM_INTERPRETER;
11785 break;
11786 }
11787 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11788 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11789 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11790 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11791 pMixedCtx->esp += cbParm;
11792 pMixedCtx->esp &= uMask;
11793 pMixedCtx->rip += pDis->cbInstr;
11794 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11795 | HM_CHANGED_GUEST_RSP
11796 | HM_CHANGED_GUEST_RFLAGS);
11797 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11798 if (fStepping)
11799 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11800
11801 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11802 break;
11803 }
11804
11805 case OP_PUSHF:
11806 {
11807 uint32_t cbParm;
11808 uint32_t uMask;
11809 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11810 {
11811 cbParm = 4;
11812 uMask = 0xffffffff;
11813 }
11814 else
11815 {
11816 cbParm = 2;
11817 uMask = 0xffff;
11818 }
11819
11820 /* Get the stack pointer & push the contents of eflags onto the stack. */
11821 RTGCPTR GCPtrStack = 0;
11822 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11823 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11824 if (RT_FAILURE(rc))
11825 {
11826 rc = VERR_EM_INTERPRETER;
11827 break;
11828 }
11829 X86EFLAGS Eflags = pMixedCtx->eflags;
11830 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11831 Eflags.Bits.u1RF = 0;
11832 Eflags.Bits.u1VM = 0;
11833
11834 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11835 if (RT_FAILURE(rc))
11836 {
11837 rc = VERR_EM_INTERPRETER;
11838 break;
11839 }
11840 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11841 pMixedCtx->esp -= cbParm;
11842 pMixedCtx->esp &= uMask;
11843 pMixedCtx->rip += pDis->cbInstr;
11844 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11845 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11846 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11847 break;
11848 }
11849
11850 case OP_IRET:
11851 {
11852 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11853 * instruction reference. */
11854 RTGCPTR GCPtrStack = 0;
11855 uint32_t uMask = 0xffff;
11856 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11857 uint16_t aIretFrame[3];
11858 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11859 {
11860 rc = VERR_EM_INTERPRETER;
11861 break;
11862 }
11863 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11864 &GCPtrStack);
11865 if (RT_SUCCESS(rc))
11866 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11867 if (RT_FAILURE(rc))
11868 {
11869 rc = VERR_EM_INTERPRETER;
11870 break;
11871 }
11872 pMixedCtx->eip = 0;
11873 pMixedCtx->ip = aIretFrame[0];
11874 pMixedCtx->cs.Sel = aIretFrame[1];
11875 pMixedCtx->cs.ValidSel = aIretFrame[1];
11876 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11877 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11878 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11879 pMixedCtx->sp += sizeof(aIretFrame);
11880 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11881 | HM_CHANGED_GUEST_SEGMENT_REGS
11882 | HM_CHANGED_GUEST_RSP
11883 | HM_CHANGED_GUEST_RFLAGS);
11884 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11885 if (fStepping)
11886 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11887 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11888 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11889 break;
11890 }
11891
11892 case OP_INT:
11893 {
11894 uint16_t uVector = pDis->Param1.uValue & 0xff;
11895 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11896 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11897 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11898 break;
11899 }
11900
11901 case OP_INTO:
11902 {
11903 if (pMixedCtx->eflags.Bits.u1OF)
11904 {
11905 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11906 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11907 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11908 }
11909 break;
11910 }
11911
11912 default:
11913 {
11914 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11915 EMCODETYPE_SUPERVISOR);
11916 rc = VBOXSTRICTRC_VAL(rc2);
11917 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11918 /** @todo We have to set pending-debug exceptions here when the guest is
11919 * single-stepping depending on the instruction that was interpreted. */
11920 Log4(("#GP rc=%Rrc\n", rc));
11921 break;
11922 }
11923 }
11924 }
11925 else
11926 rc = VERR_EM_INTERPRETER;
11927
11928 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11929 ("#GP Unexpected rc=%Rrc\n", rc));
11930 return rc;
11931}
11932
11933
11934#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11935/**
11936 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11937 * the exception reported in the VMX transient structure back into the VM.
11938 *
11939 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11940 * up-to-date.
11941 */
11942static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11943{
11944 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11945
11946 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11947 hmR0VmxCheckExitDueToEventDelivery(). */
11948 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11949 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11950 AssertRCReturn(rc, rc);
11951 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11952
11953#ifdef DEBUG_ramshankar
11954 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11955 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11956 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11957#endif
11958
11959 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11960 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11961 return VINF_SUCCESS;
11962}
11963#endif
11964
11965
11966/**
11967 * VM-exit exception handler for #PF (Page-fault exception).
11968 */
11969static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11970{
11971 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11972 PVM pVM = pVCpu->CTX_SUFF(pVM);
11973 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11974 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11975 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11976 AssertRCReturn(rc, rc);
11977
11978#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11979 if (pVM->hm.s.fNestedPaging)
11980 {
11981 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11982 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11983 {
11984 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11985 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11986 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11987 }
11988 else
11989 {
11990 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11991 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11992 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11993 }
11994 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11995 return rc;
11996 }
11997#else
11998 Assert(!pVM->hm.s.fNestedPaging);
11999 NOREF(pVM);
12000#endif
12001
12002 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12003 AssertRCReturn(rc, rc);
12004
12005 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12006 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12007
12008 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12009 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12010 (RTGCPTR)pVmxTransient->uExitQualification);
12011
12012 Log4(("#PF: rc=%Rrc\n", rc));
12013 if (rc == VINF_SUCCESS)
12014 {
12015 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12016 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12017 * memory? We don't update the whole state here... */
12018 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12019 | HM_CHANGED_GUEST_RSP
12020 | HM_CHANGED_GUEST_RFLAGS
12021 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12022 TRPMResetTrap(pVCpu);
12023 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12024 return rc;
12025 }
12026 else if (rc == VINF_EM_RAW_GUEST_TRAP)
12027 {
12028 if (!pVmxTransient->fVectoringPF)
12029 {
12030 /* It's a guest page fault and needs to be reflected to the guest. */
12031 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12032 TRPMResetTrap(pVCpu);
12033 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12034 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12035 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12036 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12037 }
12038 else
12039 {
12040 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12041 TRPMResetTrap(pVCpu);
12042 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12043 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12044 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12045 }
12046
12047 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12048 return VINF_SUCCESS;
12049 }
12050
12051 TRPMResetTrap(pVCpu);
12052 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12053 return rc;
12054}
12055
12056/** @} */
12057
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette