VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 52224

Last change on this file since 52224 was 52192, checked in by vboxsync, 10 years ago

HostDrivers/Support, VMM: support CONFIG_PAX_KERNEXEC Linux kernels
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 499.0 KB
Line 
1/* $Id: HMVMXR0.cpp 52192 2014-07-25 15:04:01Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
41# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_CHECK_GUEST_STATE
44# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
45# define HMVMX_ALWAYS_TRAP_PF
46# define HMVMX_ALWAYS_SWAP_FPU_STATE
47# define HMVMX_ALWAYS_FLUSH_TLB
48# define HMVMX_ALWAYS_SWAP_EFER
49#endif
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define HMVMX_IS_64BIT_HOST_MODE() (true)
57typedef RTHCUINTREG HMVMXHCUINTREG;
58#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
59extern "C" uint32_t g_fVMXIs64bitHost;
60# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
61typedef uint64_t HMVMXHCUINTREG;
62#else
63# define HMVMX_IS_64BIT_HOST_MODE() (false)
64typedef RTHCUINTREG HMVMXHCUINTREG;
65#endif
66
67/** Use the function table. */
68#define HMVMX_USE_FUNCTION_TABLE
69
70/** Determine which tagged-TLB flush handler to use. */
71#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
72#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
73#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
74#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
75
76/** @name Updated-guest-state flags.
77 * @{ */
78#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
79#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
80#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
81#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
82#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
83#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
84#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
85#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
86#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
87#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
88#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
89#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
90#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
91#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
92#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
93#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
94#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
95#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
96#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
97#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
98#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
99 | HMVMX_UPDATED_GUEST_RSP \
100 | HMVMX_UPDATED_GUEST_RFLAGS \
101 | HMVMX_UPDATED_GUEST_CR0 \
102 | HMVMX_UPDATED_GUEST_CR3 \
103 | HMVMX_UPDATED_GUEST_CR4 \
104 | HMVMX_UPDATED_GUEST_GDTR \
105 | HMVMX_UPDATED_GUEST_IDTR \
106 | HMVMX_UPDATED_GUEST_LDTR \
107 | HMVMX_UPDATED_GUEST_TR \
108 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
109 | HMVMX_UPDATED_GUEST_DEBUG \
110 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
112 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
113 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
114 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
115 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
116 | HMVMX_UPDATED_GUEST_INTR_STATE \
117 | HMVMX_UPDATED_GUEST_APIC_STATE)
118/** @} */
119
120/** @name
121 * Flags to skip redundant reads of some common VMCS fields that are not part of
122 * the guest-CPU state but are in the transient structure.
123 */
124#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
125#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
131/** @} */
132
133/** @name
134 * States of the VMCS.
135 *
136 * This does not reflect all possible VMCS states but currently only those
137 * needed for maintaining the VMCS consistently even when thread-context hooks
138 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
139 */
140#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
141#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
142#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
143/** @} */
144
145/**
146 * Exception bitmap mask for real-mode guests (real-on-v86).
147 *
148 * We need to intercept all exceptions manually (except #PF). #NM is also
149 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
150 * even in real-mode if we have Nested Paging support.
151 */
152#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
153 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
154 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
155 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
156 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
157 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
158 | RT_BIT(X86_XCPT_XF))
159
160/**
161 * Exception bitmap mask for all contributory exceptions.
162 *
163 * Page fault is deliberately excluded here as it's conditional as to whether
164 * it's contributory or benign. Page faults are handled separately.
165 */
166#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
167 | RT_BIT(X86_XCPT_DE))
168
169/** Maximum VM-instruction error number. */
170#define HMVMX_INSTR_ERROR_MAX 28
171
172/** Profiling macro. */
173#ifdef HM_PROFILE_EXIT_DISPATCH
174# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
176#else
177# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
178# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
179#endif
180
181/** Assert that preemption is disabled or covered by thread-context hooks. */
182#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
184
185/** Assert that we haven't migrated CPUs when thread-context hooks are not
186 * used. */
187#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
188 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
189 ("Illegal migration! Entered on CPU %u Current %u\n", \
190 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
191
192/** Helper macro for VM-exit handlers called unexpectedly. */
193#define HMVMX_RETURN_UNEXPECTED_EXIT() \
194 do { \
195 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
196 return VERR_VMX_UNEXPECTED_EXIT; \
197 } while (0)
198
199
200/*******************************************************************************
201* Structures and Typedefs *
202*******************************************************************************/
203/**
204 * VMX transient state.
205 *
206 * A state structure for holding miscellaneous information across
207 * VMX non-root operation and restored after the transition.
208 */
209typedef struct VMXTRANSIENT
210{
211 /** The host's rflags/eflags. */
212 RTCCUINTREG uEflags;
213#if HC_ARCH_BITS == 32
214 uint32_t u32Alignment0;
215#endif
216 /** The guest's TPR value used for TPR shadowing. */
217 uint8_t u8GuestTpr;
218 /** Alignment. */
219 uint8_t abAlignment0[7];
220
221 /** The basic VM-exit reason. */
222 uint16_t uExitReason;
223 /** Alignment. */
224 uint16_t u16Alignment0;
225 /** The VM-exit interruption error code. */
226 uint32_t uExitIntErrorCode;
227 /** The VM-exit exit qualification. */
228 uint64_t uExitQualification;
229
230 /** The VM-exit interruption-information field. */
231 uint32_t uExitIntInfo;
232 /** The VM-exit instruction-length field. */
233 uint32_t cbInstr;
234 /** The VM-exit instruction-information field. */
235 union
236 {
237 /** Plain unsigned int representation. */
238 uint32_t u;
239 /** INS and OUTS information. */
240 struct
241 {
242 uint32_t u6Reserved0 : 7;
243 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
244 uint32_t u3AddrSize : 3;
245 uint32_t u5Reserved1 : 5;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 uint32_t uReserved2 : 14;
249 } StrIo;
250 } ExitInstrInfo;
251 /** Whether the VM-entry failed or not. */
252 bool fVMEntryFailed;
253 /** Alignment. */
254 uint8_t abAlignment1[3];
255
256 /** The VM-entry interruption-information field. */
257 uint32_t uEntryIntInfo;
258 /** The VM-entry exception error code field. */
259 uint32_t uEntryXcptErrorCode;
260 /** The VM-entry instruction length field. */
261 uint32_t cbEntryInstr;
262
263 /** IDT-vectoring information field. */
264 uint32_t uIdtVectoringInfo;
265 /** IDT-vectoring error code. */
266 uint32_t uIdtVectoringErrorCode;
267
268 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
269 uint32_t fVmcsFieldsRead;
270
271 /** Whether the guest FPU was active at the time of VM-exit. */
272 bool fWasGuestFPUStateActive;
273 /** Whether the guest debug state was active at the time of VM-exit. */
274 bool fWasGuestDebugStateActive;
275 /** Whether the hyper debug state was active at the time of VM-exit. */
276 bool fWasHyperDebugStateActive;
277 /** Whether TSC-offsetting should be setup before VM-entry. */
278 bool fUpdateTscOffsettingAndPreemptTimer;
279 /** Whether the VM-exit was caused by a page-fault during delivery of a
280 * contributory exception or a page-fault. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns VBox status code.
323 * @param pVCpu Pointer to the VMCPU.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337
338/*******************************************************************************
339* Internal Functions *
340*******************************************************************************/
341static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
342static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
343static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
344 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
345#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
346static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
347#endif
348#ifndef HMVMX_USE_FUNCTION_TABLE
349DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
350# define HMVMX_EXIT_DECL static int
351#else
352# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
353#endif
354
355/** @name VM-exit handlers.
356 * @{
357 */
358static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
359static FNVMXEXITHANDLER hmR0VmxExitExtInt;
360static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
361static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
362static FNVMXEXITHANDLER hmR0VmxExitSipi;
363static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
364static FNVMXEXITHANDLER hmR0VmxExitSmi;
365static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
366static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
367static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
368static FNVMXEXITHANDLER hmR0VmxExitCpuid;
369static FNVMXEXITHANDLER hmR0VmxExitGetsec;
370static FNVMXEXITHANDLER hmR0VmxExitHlt;
371static FNVMXEXITHANDLER hmR0VmxExitInvd;
372static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
373static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
374static FNVMXEXITHANDLER hmR0VmxExitVmcall;
375static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
376static FNVMXEXITHANDLER hmR0VmxExitRsm;
377static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
378static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
379static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
380static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
381static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
382static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
383static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
384static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
385static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
386static FNVMXEXITHANDLER hmR0VmxExitMwait;
387static FNVMXEXITHANDLER hmR0VmxExitMtf;
388static FNVMXEXITHANDLER hmR0VmxExitMonitor;
389static FNVMXEXITHANDLER hmR0VmxExitPause;
390static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
391static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
392static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
393static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
394static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
395static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
396static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
397static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
398static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
399static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
400static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
401static FNVMXEXITHANDLER hmR0VmxExitRdrand;
402static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
403/** @} */
404
405static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
412static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413#endif
414static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
415
416/*******************************************************************************
417* Global Variables *
418*******************************************************************************/
419#ifdef HMVMX_USE_FUNCTION_TABLE
420
421/**
422 * VMX_EXIT dispatch table.
423 */
424static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
425{
426 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
427 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
428 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
429 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
430 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
431 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
432 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
433 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
434 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
435 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
436 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
437 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
438 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
439 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
440 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
441 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
442 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
443 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
444 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
445 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
446 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
447 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
448 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
449 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
450 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
451 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
452 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
453 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
454 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
455 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
456 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
457 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
458 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
459 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
460 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
461 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
462 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
463 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
464 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
465 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
466 /* 40 UNDEFINED */ hmR0VmxExitPause,
467 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
468 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
469 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
470 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
471 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
472 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
473 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
474 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
475 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
476 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
477 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
478 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
479 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
480 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
481 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
482 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
483 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
484 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
485 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
486};
487#endif /* HMVMX_USE_FUNCTION_TABLE */
488
489#ifdef VBOX_STRICT
490static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
491{
492 /* 0 */ "(Not Used)",
493 /* 1 */ "VMCALL executed in VMX root operation.",
494 /* 2 */ "VMCLEAR with invalid physical address.",
495 /* 3 */ "VMCLEAR with VMXON pointer.",
496 /* 4 */ "VMLAUNCH with non-clear VMCS.",
497 /* 5 */ "VMRESUME with non-launched VMCS.",
498 /* 6 */ "VMRESUME after VMXOFF",
499 /* 7 */ "VM entry with invalid control fields.",
500 /* 8 */ "VM entry with invalid host state fields.",
501 /* 9 */ "VMPTRLD with invalid physical address.",
502 /* 10 */ "VMPTRLD with VMXON pointer.",
503 /* 11 */ "VMPTRLD with incorrect revision identifier.",
504 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
505 /* 13 */ "VMWRITE to read-only VMCS component.",
506 /* 14 */ "(Not Used)",
507 /* 15 */ "VMXON executed in VMX root operation.",
508 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
509 /* 17 */ "VM entry with non-launched executing VMCS.",
510 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
511 /* 19 */ "VMCALL with non-clear VMCS.",
512 /* 20 */ "VMCALL with invalid VM-exit control fields.",
513 /* 21 */ "(Not Used)",
514 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
515 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
516 /* 24 */ "VMCALL with invalid SMM-monitor features.",
517 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
518 /* 26 */ "VM entry with events blocked by MOV SS.",
519 /* 27 */ "(Not Used)",
520 /* 28 */ "Invalid operand to INVEPT/INVVPID."
521};
522#endif /* VBOX_STRICT */
523
524
525
526/**
527 * Updates the VM's last error record. If there was a VMX instruction error,
528 * reads the error data from the VMCS and updates VCPU's last error record as
529 * well.
530 *
531 * @param pVM Pointer to the VM.
532 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
533 * VERR_VMX_UNABLE_TO_START_VM or
534 * VERR_VMX_INVALID_VMCS_FIELD).
535 * @param rc The error code.
536 */
537static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
538{
539 AssertPtr(pVM);
540 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
541 || rc == VERR_VMX_UNABLE_TO_START_VM)
542 {
543 AssertPtrReturnVoid(pVCpu);
544 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
545 }
546 pVM->hm.s.lLastError = rc;
547}
548
549
550/**
551 * Reads the VM-entry interruption-information field from the VMCS into the VMX
552 * transient structure.
553 *
554 * @returns VBox status code.
555 * @param pVmxTransient Pointer to the VMX transient structure.
556 *
557 * @remarks No-long-jump zone!!!
558 */
559DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
560{
561 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
562 AssertRCReturn(rc, rc);
563 return VINF_SUCCESS;
564}
565
566
567/**
568 * Reads the VM-entry exception error code field from the VMCS into
569 * the VMX transient structure.
570 *
571 * @returns VBox status code.
572 * @param pVmxTransient Pointer to the VMX transient structure.
573 *
574 * @remarks No-long-jump zone!!!
575 */
576DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
577{
578 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
579 AssertRCReturn(rc, rc);
580 return VINF_SUCCESS;
581}
582
583
584/**
585 * Reads the VM-entry exception error code field from the VMCS into
586 * the VMX transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600
601/**
602 * Reads the VM-exit interruption-information field from the VMCS into the VMX
603 * transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 */
608DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
611 {
612 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
613 AssertRCReturn(rc, rc);
614 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
615 }
616 return VINF_SUCCESS;
617}
618
619
620/**
621 * Reads the VM-exit interruption error code from the VMCS into the VMX
622 * transient structure.
623 *
624 * @returns VBox status code.
625 * @param pVmxTransient Pointer to the VMX transient structure.
626 */
627DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
630 {
631 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
632 AssertRCReturn(rc, rc);
633 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
634 }
635 return VINF_SUCCESS;
636}
637
638
639/**
640 * Reads the VM-exit instruction length field from the VMCS into the VMX
641 * transient structure.
642 *
643 * @returns VBox status code.
644 * @param pVCpu Pointer to the VMCPU.
645 * @param pVmxTransient Pointer to the VMX transient structure.
646 */
647DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
648{
649 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
650 {
651 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
652 AssertRCReturn(rc, rc);
653 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
654 }
655 return VINF_SUCCESS;
656}
657
658
659/**
660 * Reads the VM-exit instruction-information field from the VMCS into
661 * the VMX transient structure.
662 *
663 * @returns VBox status code.
664 * @param pVmxTransient Pointer to the VMX transient structure.
665 */
666DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
667{
668 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
669 {
670 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
671 AssertRCReturn(rc, rc);
672 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
673 }
674 return VINF_SUCCESS;
675}
676
677
678/**
679 * Reads the exit qualification from the VMCS into the VMX transient structure.
680 *
681 * @returns VBox status code.
682 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
683 * case).
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
689 {
690 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the IDT-vectoring information field from the VMCS into the VMX
700 * transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 *
705 * @remarks No-long-jump zone!!!
706 */
707DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
710 {
711 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the IDT-vectoring error code from the VMCS into the VMX
721 * transient structure.
722 *
723 * @returns VBox status code.
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 */
726DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
727{
728 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
729 {
730 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
731 AssertRCReturn(rc, rc);
732 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
733 }
734 return VINF_SUCCESS;
735}
736
737
738/**
739 * Enters VMX root mode operation on the current CPU.
740 *
741 * @returns VBox status code.
742 * @param pVM Pointer to the VM (optional, can be NULL, after
743 * a resume).
744 * @param HCPhysCpuPage Physical address of the VMXON region.
745 * @param pvCpuPage Pointer to the VMXON region.
746 */
747static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
748{
749 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
750 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
751 Assert(pvCpuPage);
752 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
753
754 if (pVM)
755 {
756 /* Write the VMCS revision dword to the VMXON region. */
757 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
758 }
759
760 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
761 RTCCUINTREG uEflags = ASMIntDisableFlags();
762
763 /* Enable the VMX bit in CR4 if necessary. */
764 RTCCUINTREG uCr4 = ASMGetCR4();
765 if (!(uCr4 & X86_CR4_VMXE))
766 ASMSetCR4(uCr4 | X86_CR4_VMXE);
767
768 /* Enter VMX root mode. */
769 int rc = VMXEnable(HCPhysCpuPage);
770 if (RT_FAILURE(rc))
771 ASMSetCR4(uCr4);
772
773 /* Restore interrupts. */
774 ASMSetFlags(uEflags);
775 return rc;
776}
777
778
779/**
780 * Exits VMX root mode operation on the current CPU.
781 *
782 * @returns VBox status code.
783 */
784static int hmR0VmxLeaveRootMode(void)
785{
786 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
787
788 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
789 RTCCUINTREG uEflags = ASMIntDisableFlags();
790
791 /* If we're for some reason not in VMX root mode, then don't leave it. */
792 RTCCUINTREG uHostCR4 = ASMGetCR4();
793
794 int rc;
795 if (uHostCR4 & X86_CR4_VMXE)
796 {
797 /* Exit VMX root mode and clear the VMX bit in CR4. */
798 VMXDisable();
799 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
800 rc = VINF_SUCCESS;
801 }
802 else
803 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
804
805 /* Restore interrupts. */
806 ASMSetFlags(uEflags);
807 return rc;
808}
809
810
811/**
812 * Allocates and maps one physically contiguous page. The allocated page is
813 * zero'd out. (Used by various VT-x structures).
814 *
815 * @returns IPRT status code.
816 * @param pMemObj Pointer to the ring-0 memory object.
817 * @param ppVirt Where to store the virtual address of the
818 * allocation.
819 * @param pPhys Where to store the physical address of the
820 * allocation.
821 */
822DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
823{
824 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
825 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
826 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
827
828 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
829 if (RT_FAILURE(rc))
830 return rc;
831 *ppVirt = RTR0MemObjAddress(*pMemObj);
832 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
833 ASMMemZero32(*ppVirt, PAGE_SIZE);
834 return VINF_SUCCESS;
835}
836
837
838/**
839 * Frees and unmaps an allocated physical page.
840 *
841 * @param pMemObj Pointer to the ring-0 memory object.
842 * @param ppVirt Where to re-initialize the virtual address of
843 * allocation as 0.
844 * @param pHCPhys Where to re-initialize the physical address of the
845 * allocation as 0.
846 */
847DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
848{
849 AssertPtr(pMemObj);
850 AssertPtr(ppVirt);
851 AssertPtr(pHCPhys);
852 if (*pMemObj != NIL_RTR0MEMOBJ)
853 {
854 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
855 AssertRC(rc);
856 *pMemObj = NIL_RTR0MEMOBJ;
857 *ppVirt = 0;
858 *pHCPhys = 0;
859 }
860}
861
862
863/**
864 * Worker function to free VT-x related structures.
865 *
866 * @returns IPRT status code.
867 * @param pVM Pointer to the VM.
868 */
869static void hmR0VmxStructsFree(PVM pVM)
870{
871 for (VMCPUID i = 0; i < pVM->cCpus; i++)
872 {
873 PVMCPU pVCpu = &pVM->aCpus[i];
874 AssertPtr(pVCpu);
875
876 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
877 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
878
879 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
880 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
881
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
883 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
884 }
885
886 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
887#ifdef VBOX_WITH_CRASHDUMP_MAGIC
888 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
889#endif
890}
891
892
893/**
894 * Worker function to allocate VT-x related VM structures.
895 *
896 * @returns IPRT status code.
897 * @param pVM Pointer to the VM.
898 */
899static int hmR0VmxStructsAlloc(PVM pVM)
900{
901 /*
902 * Initialize members up-front so we can cleanup properly on allocation failure.
903 */
904#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
905 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907 pVM->hm.s.vmx.HCPhys##a_Name = 0;
908
909#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
910 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
911 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
912 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
913
914#ifdef VBOX_WITH_CRASHDUMP_MAGIC
915 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
916#endif
917 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
918
919 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
920 for (VMCPUID i = 0; i < pVM->cCpus; i++)
921 {
922 PVMCPU pVCpu = &pVM->aCpus[i];
923 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
924 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
925 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
926 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
927 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
928 }
929#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
930#undef VMXLOCAL_INIT_VM_MEMOBJ
931
932 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
933 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
934 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
935 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
936
937 /*
938 * Allocate all the VT-x structures.
939 */
940 int rc = VINF_SUCCESS;
941#ifdef VBOX_WITH_CRASHDUMP_MAGIC
942 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
943 if (RT_FAILURE(rc))
944 goto cleanup;
945 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
946 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
947#endif
948
949 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
950 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
951 {
952 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
953 &pVM->hm.s.vmx.HCPhysApicAccess);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956 }
957
958 /*
959 * Initialize per-VCPU VT-x structures.
960 */
961 for (VMCPUID i = 0; i < pVM->cCpus; i++)
962 {
963 PVMCPU pVCpu = &pVM->aCpus[i];
964 AssertPtr(pVCpu);
965
966 /* Allocate the VM control structure (VMCS). */
967 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
968 if (RT_FAILURE(rc))
969 goto cleanup;
970
971 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
972 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
973 {
974 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
975 &pVCpu->hm.s.vmx.HCPhysVirtApic);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978 }
979
980 /*
981 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
982 * transparent accesses of specific MSRs.
983 *
984 * If the condition for enabling MSR bitmaps changes here, don't forget to
985 * update HMIsMsrBitmapsAvailable().
986 */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
988 {
989 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
990 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
994 }
995
996 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
997 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000
1001 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1002 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1003 if (RT_FAILURE(rc))
1004 goto cleanup;
1005 }
1006
1007 return VINF_SUCCESS;
1008
1009cleanup:
1010 hmR0VmxStructsFree(pVM);
1011 return rc;
1012}
1013
1014
1015/**
1016 * Does global VT-x initialization (called during module initialization).
1017 *
1018 * @returns VBox status code.
1019 */
1020VMMR0DECL(int) VMXR0GlobalInit(void)
1021{
1022#ifdef HMVMX_USE_FUNCTION_TABLE
1023 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1024# ifdef VBOX_STRICT
1025 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1026 Assert(g_apfnVMExitHandlers[i]);
1027# endif
1028#endif
1029 return VINF_SUCCESS;
1030}
1031
1032
1033/**
1034 * Does global VT-x termination (called during module termination).
1035 */
1036VMMR0DECL(void) VMXR0GlobalTerm()
1037{
1038 /* Nothing to do currently. */
1039}
1040
1041
1042/**
1043 * Sets up and activates VT-x on the current CPU.
1044 *
1045 * @returns VBox status code.
1046 * @param pCpu Pointer to the global CPU info struct.
1047 * @param pVM Pointer to the VM (can be NULL after a host resume
1048 * operation).
1049 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1050 * fEnabledByHost is true).
1051 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1052 * @a fEnabledByHost is true).
1053 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1054 * enable VT-x on the host.
1055 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1056 */
1057VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1058 void *pvMsrs)
1059{
1060 Assert(pCpu);
1061 Assert(pvMsrs);
1062 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1063
1064 /* Enable VT-x if it's not already enabled by the host. */
1065 if (!fEnabledByHost)
1066 {
1067 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1068 if (RT_FAILURE(rc))
1069 return rc;
1070 }
1071
1072 /*
1073 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1074 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1075 */
1076 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1077 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1078 {
1079 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1080 pCpu->fFlushAsidBeforeUse = false;
1081 }
1082 else
1083 pCpu->fFlushAsidBeforeUse = true;
1084
1085 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1086 ++pCpu->cTlbFlushes;
1087
1088 return VINF_SUCCESS;
1089}
1090
1091
1092/**
1093 * Deactivates VT-x on the current CPU.
1094 *
1095 * @returns VBox status code.
1096 * @param pCpu Pointer to the global CPU info struct.
1097 * @param pvCpuPage Pointer to the VMXON region.
1098 * @param HCPhysCpuPage Physical address of the VMXON region.
1099 *
1100 * @remarks This function should never be called when SUPR0EnableVTx() or
1101 * similar was used to enable VT-x on the host.
1102 */
1103VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1104{
1105 NOREF(pCpu);
1106 NOREF(pvCpuPage);
1107 NOREF(HCPhysCpuPage);
1108
1109 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1110 return hmR0VmxLeaveRootMode();
1111}
1112
1113
1114/**
1115 * Sets the permission bits for the specified MSR in the MSR bitmap.
1116 *
1117 * @param pVCpu Pointer to the VMCPU.
1118 * @param uMSR The MSR value.
1119 * @param enmRead Whether reading this MSR causes a VM-exit.
1120 * @param enmWrite Whether writing this MSR causes a VM-exit.
1121 */
1122static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1123{
1124 int32_t iBit;
1125 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1126
1127 /*
1128 * Layout:
1129 * 0x000 - 0x3ff - Low MSR read bits
1130 * 0x400 - 0x7ff - High MSR read bits
1131 * 0x800 - 0xbff - Low MSR write bits
1132 * 0xc00 - 0xfff - High MSR write bits
1133 */
1134 if (uMsr <= 0x00001FFF)
1135 iBit = uMsr;
1136 else if ( uMsr >= 0xC0000000
1137 && uMsr <= 0xC0001FFF)
1138 {
1139 iBit = (uMsr - 0xC0000000);
1140 pbMsrBitmap += 0x400;
1141 }
1142 else
1143 {
1144 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1145 return;
1146 }
1147
1148 Assert(iBit <= 0x1fff);
1149 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1150 ASMBitSet(pbMsrBitmap, iBit);
1151 else
1152 ASMBitClear(pbMsrBitmap, iBit);
1153
1154 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1155 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1156 else
1157 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1158}
1159
1160
1161#ifdef VBOX_STRICT
1162/**
1163 * Gets the permission bits for the specified MSR in the MSR bitmap.
1164 *
1165 * @returns VBox status code.
1166 * @retval VINF_SUCCESS if the specified MSR is found.
1167 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1168 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1169 *
1170 * @param pVCpu Pointer to the VMCPU.
1171 * @param uMsr The MSR.
1172 * @param penmRead Where to store the read permissions.
1173 * @param penmWrite Where to store the write permissions.
1174 */
1175static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1176{
1177 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1178 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1179 int32_t iBit;
1180 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1181
1182 /* See hmR0VmxSetMsrPermission() for the layout. */
1183 if (uMsr <= 0x00001FFF)
1184 iBit = uMsr;
1185 else if ( uMsr >= 0xC0000000
1186 && uMsr <= 0xC0001FFF)
1187 {
1188 iBit = (uMsr - 0xC0000000);
1189 pbMsrBitmap += 0x400;
1190 }
1191 else
1192 {
1193 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1194 return VERR_NOT_SUPPORTED;
1195 }
1196
1197 Assert(iBit <= 0x1fff);
1198 if (ASMBitTest(pbMsrBitmap, iBit))
1199 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1200 else
1201 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1202
1203 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1204 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1205 else
1206 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1207 return VINF_SUCCESS;
1208}
1209#endif /* VBOX_STRICT */
1210
1211
1212/**
1213 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1214 * area.
1215 *
1216 * @returns VBox status code.
1217 * @param pVCpu Pointer to the VMCPU.
1218 * @param cMsrs The number of MSRs.
1219 */
1220DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1221{
1222 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1223 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1224 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1225 {
1226 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1227 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1228 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1229 }
1230
1231 /* Update number of guest MSRs to load/store across the world-switch. */
1232 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1233 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1234
1235 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1236 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237
1238 /* Update the VCPU's copy of the MSR count. */
1239 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1240
1241 return VINF_SUCCESS;
1242}
1243
1244
1245/**
1246 * Adds a new (or updates the value of an existing) guest/host MSR
1247 * pair to be swapped during the world-switch as part of the
1248 * auto-load/store MSR area in the VMCS.
1249 *
1250 * @returns true if the MSR was added -and- its value was updated, false
1251 * otherwise.
1252 * @param pVCpu Pointer to the VMCPU.
1253 * @param uMsr The MSR.
1254 * @param uGuestMsr Value of the guest MSR.
1255 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1256 * necessary.
1257 */
1258static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1259{
1260 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1262 uint32_t i;
1263 for (i = 0; i < cMsrs; i++)
1264 {
1265 if (pGuestMsr->u32Msr == uMsr)
1266 break;
1267 pGuestMsr++;
1268 }
1269
1270 bool fAdded = false;
1271 if (i == cMsrs)
1272 {
1273 ++cMsrs;
1274 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1275 AssertRC(rc);
1276
1277 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1278 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1279 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1280
1281 fAdded = true;
1282 }
1283
1284 /* Update the MSR values in the auto-load/store MSR area. */
1285 pGuestMsr->u32Msr = uMsr;
1286 pGuestMsr->u64Value = uGuestMsrValue;
1287
1288 /* Create/update the MSR slot in the host MSR area. */
1289 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1290 pHostMsr += i;
1291 pHostMsr->u32Msr = uMsr;
1292
1293 /*
1294 * Update the host MSR only when requested by the caller AND when we're
1295 * adding it to the auto-load/store area. Otherwise, it would have been
1296 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1297 */
1298 bool fUpdatedMsrValue = false;
1299 if ( fAdded
1300 && fUpdateHostMsr)
1301 {
1302 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1303 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1304 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1305 fUpdatedMsrValue = true;
1306 }
1307
1308 return fUpdatedMsrValue;
1309}
1310
1311
1312/**
1313 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1314 * auto-load/store MSR area in the VMCS.
1315 *
1316 * @returns VBox status code.
1317 * @param pVCpu Pointer to the VMCPU.
1318 * @param uMsr The MSR.
1319 */
1320static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1321{
1322 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1323 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1324 for (uint32_t i = 0; i < cMsrs; i++)
1325 {
1326 /* Find the MSR. */
1327 if (pGuestMsr->u32Msr == uMsr)
1328 {
1329 /* If it's the last MSR, simply reduce the count. */
1330 if (i == cMsrs - 1)
1331 {
1332 --cMsrs;
1333 break;
1334 }
1335
1336 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1337 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1338 pLastGuestMsr += cMsrs - 1;
1339 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1340 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1341
1342 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1343 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1344 pLastHostMsr += cMsrs - 1;
1345 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1346 pHostMsr->u64Value = pLastHostMsr->u64Value;
1347 --cMsrs;
1348 break;
1349 }
1350 pGuestMsr++;
1351 }
1352
1353 /* Update the VMCS if the count changed (meaning the MSR was found). */
1354 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1355 {
1356 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1357 AssertRCReturn(rc, rc);
1358
1359 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1360 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1361 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1362
1363 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1364 return VINF_SUCCESS;
1365 }
1366
1367 return VERR_NOT_FOUND;
1368}
1369
1370
1371/**
1372 * Checks if the specified guest MSR is part of the auto-load/store area in
1373 * the VMCS.
1374 *
1375 * @returns true if found, false otherwise.
1376 * @param pVCpu Pointer to the VMCPU.
1377 * @param uMsr The MSR to find.
1378 */
1379static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1380{
1381 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1382 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1383
1384 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1385 {
1386 if (pGuestMsr->u32Msr == uMsr)
1387 return true;
1388 }
1389 return false;
1390}
1391
1392
1393/**
1394 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1395 *
1396 * @param pVCpu Pointer to the VMCPU.
1397 *
1398 * @remarks No-long-jump zone!!!
1399 */
1400static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1401{
1402 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1403 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1405 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1406
1407 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1408 {
1409 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1410
1411 /*
1412 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1413 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1414 */
1415 if (pHostMsr->u32Msr == MSR_K6_EFER)
1416 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1417 else
1418 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1419 }
1420
1421 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1422}
1423
1424
1425#if HC_ARCH_BITS == 64
1426/**
1427 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1428 * perform lazy restoration of the host MSRs while leaving VT-x.
1429 *
1430 * @param pVCpu Pointer to the VMCPU.
1431 *
1432 * @remarks No-long-jump zone!!!
1433 */
1434static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1435{
1436 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1437
1438 /*
1439 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1440 */
1441 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1442 {
1443 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1444 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1445 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1446 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1447 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1448 }
1449}
1450
1451
1452/**
1453 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1454 * lazily while leaving VT-x.
1455 *
1456 * @returns true if it does, false otherwise.
1457 * @param pVCpu Pointer to the VMCPU.
1458 * @param uMsr The MSR to check.
1459 */
1460static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1461{
1462 NOREF(pVCpu);
1463 switch (uMsr)
1464 {
1465 case MSR_K8_LSTAR:
1466 case MSR_K6_STAR:
1467 case MSR_K8_SF_MASK:
1468 case MSR_K8_KERNEL_GS_BASE:
1469 return true;
1470 }
1471 return false;
1472}
1473
1474
1475/**
1476 * Saves a set of guests MSRs back into the guest-CPU context.
1477 *
1478 * @param pVCpu Pointer to the VMCPU.
1479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1480 * out-of-sync. Make sure to update the required fields
1481 * before using them.
1482 *
1483 * @remarks No-long-jump zone!!!
1484 */
1485static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1486{
1487 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1488 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1489
1490 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1491 {
1492 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1493 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1494 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1495 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1496 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1497 }
1498}
1499
1500
1501/**
1502 * Loads a set of guests MSRs to allow read/passthru to the guest.
1503 *
1504 * The name of this function is slightly confusing. This function does NOT
1505 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1506 * common prefix for functions dealing with "lazy restoration" of the shared
1507 * MSRs.
1508 *
1509 * @param pVCpu Pointer to the VMCPU.
1510 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1511 * out-of-sync. Make sure to update the required fields
1512 * before using them.
1513 *
1514 * @remarks No-long-jump zone!!!
1515 */
1516static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1517{
1518 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1519 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1520
1521 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1522 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1523 {
1524#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1525 do { \
1526 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1527 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1528 else \
1529 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1530 } while (0)
1531
1532 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1533 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1534 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1535 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1536#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1537 }
1538 else
1539 {
1540 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1541 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1542 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1543 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1544 }
1545 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1546}
1547
1548
1549/**
1550 * Performs lazy restoration of the set of host MSRs if they were previously
1551 * loaded with guest MSR values.
1552 *
1553 * @param pVCpu Pointer to the VMCPU.
1554 *
1555 * @remarks No-long-jump zone!!!
1556 * @remarks The guest MSRs should have been saved back into the guest-CPU
1557 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1558 */
1559static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1560{
1561 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1562 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1563
1564 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1565 {
1566 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1567 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1568 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1569 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1570 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1571 }
1572 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1573}
1574#endif /* HC_ARCH_BITS == 64 */
1575
1576
1577/**
1578 * Verifies that our cached values of the VMCS controls are all
1579 * consistent with what's actually present in the VMCS.
1580 *
1581 * @returns VBox status code.
1582 * @param pVCpu Pointer to the VMCPU.
1583 */
1584static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1585{
1586 uint32_t u32Val;
1587 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1588 AssertRCReturn(rc, rc);
1589 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1590 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1591
1592 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1593 AssertRCReturn(rc, rc);
1594 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1595 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1596
1597 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1598 AssertRCReturn(rc, rc);
1599 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1600 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1601
1602 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1603 AssertRCReturn(rc, rc);
1604 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1605 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1606
1607 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1608 AssertRCReturn(rc, rc);
1609 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1610 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1611
1612 return VINF_SUCCESS;
1613}
1614
1615
1616#ifdef VBOX_STRICT
1617/**
1618 * Verifies that our cached host EFER value has not changed
1619 * since we cached it.
1620 *
1621 * @param pVCpu Pointer to the VMCPU.
1622 */
1623static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1624{
1625 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1626
1627 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1628 {
1629 uint64_t u64Val;
1630 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1631 AssertRC(rc);
1632
1633 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1634 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1635 }
1636}
1637
1638
1639/**
1640 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1641 * VMCS are correct.
1642 *
1643 * @param pVCpu Pointer to the VMCPU.
1644 */
1645static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1646{
1647 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1648
1649 /* Verify MSR counts in the VMCS are what we think it should be. */
1650 uint32_t cMsrs;
1651 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1652 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1653
1654 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1655 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1656
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1658 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1659
1660 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1661 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1662 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1663 {
1664 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1665 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1666 pGuestMsr->u32Msr, cMsrs));
1667
1668 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1669 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1670 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1671
1672 /* Verify that the permissions are as expected in the MSR bitmap. */
1673 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1674 {
1675 VMXMSREXITREAD enmRead;
1676 VMXMSREXITWRITE enmWrite;
1677 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1678 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1679 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1680 {
1681 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1682 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1683 }
1684 else
1685 {
1686 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1687 pGuestMsr->u32Msr, cMsrs));
1688 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1689 pGuestMsr->u32Msr, cMsrs));
1690 }
1691 }
1692 }
1693}
1694#endif /* VBOX_STRICT */
1695
1696
1697/**
1698 * Flushes the TLB using EPT.
1699 *
1700 * @returns VBox status code.
1701 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1702 * enmFlush).
1703 * @param enmFlush Type of flush.
1704 *
1705 * @remarks Caller is responsible for making sure this function is called only
1706 * when NestedPaging is supported and providing @a enmFlush that is
1707 * supported by the CPU.
1708 * @remarks Can be called with interrupts disabled.
1709 */
1710static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1711{
1712 uint64_t au64Descriptor[2];
1713 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1714 au64Descriptor[0] = 0;
1715 else
1716 {
1717 Assert(pVCpu);
1718 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1719 }
1720 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1721
1722 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1723 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1724 rc));
1725 if ( RT_SUCCESS(rc)
1726 && pVCpu)
1727 {
1728 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1729 }
1730}
1731
1732
1733/**
1734 * Flushes the TLB using VPID.
1735 *
1736 * @returns VBox status code.
1737 * @param pVM Pointer to the VM.
1738 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1739 * enmFlush).
1740 * @param enmFlush Type of flush.
1741 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1742 * on @a enmFlush).
1743 *
1744 * @remarks Can be called with interrupts disabled.
1745 */
1746static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1747{
1748 NOREF(pVM);
1749 AssertPtr(pVM);
1750 Assert(pVM->hm.s.vmx.fVpid);
1751
1752 uint64_t au64Descriptor[2];
1753 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1754 {
1755 au64Descriptor[0] = 0;
1756 au64Descriptor[1] = 0;
1757 }
1758 else
1759 {
1760 AssertPtr(pVCpu);
1761 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1762 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1763 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1764 au64Descriptor[1] = GCPtr;
1765 }
1766
1767 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1768 AssertMsg(rc == VINF_SUCCESS,
1769 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1770 if ( RT_SUCCESS(rc)
1771 && pVCpu)
1772 {
1773 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1774 }
1775}
1776
1777
1778/**
1779 * Invalidates a guest page by guest virtual address. Only relevant for
1780 * EPT/VPID, otherwise there is nothing really to invalidate.
1781 *
1782 * @returns VBox status code.
1783 * @param pVM Pointer to the VM.
1784 * @param pVCpu Pointer to the VMCPU.
1785 * @param GCVirt Guest virtual address of the page to invalidate.
1786 */
1787VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1788{
1789 AssertPtr(pVM);
1790 AssertPtr(pVCpu);
1791 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1792
1793 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1794 if (!fFlushPending)
1795 {
1796 /*
1797 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1798 * See @bugref{6043} and @bugref{6177}.
1799 *
1800 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1801 * function maybe called in a loop with individual addresses.
1802 */
1803 if (pVM->hm.s.vmx.fVpid)
1804 {
1805 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1806 {
1807 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1808 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1809 }
1810 else
1811 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1812 }
1813 else if (pVM->hm.s.fNestedPaging)
1814 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1815 }
1816
1817 return VINF_SUCCESS;
1818}
1819
1820
1821/**
1822 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1823 * otherwise there is nothing really to invalidate.
1824 *
1825 * @returns VBox status code.
1826 * @param pVM Pointer to the VM.
1827 * @param pVCpu Pointer to the VMCPU.
1828 * @param GCPhys Guest physical address of the page to invalidate.
1829 */
1830VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1831{
1832 NOREF(pVM); NOREF(GCPhys);
1833 LogFlowFunc(("%RGp\n", GCPhys));
1834
1835 /*
1836 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1837 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1838 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1839 */
1840 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1841 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1842 return VINF_SUCCESS;
1843}
1844
1845
1846/**
1847 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1848 * case where neither EPT nor VPID is supported by the CPU.
1849 *
1850 * @param pVM Pointer to the VM.
1851 * @param pVCpu Pointer to the VMCPU.
1852 * @param pCpu Pointer to the global HM struct.
1853 *
1854 * @remarks Called with interrupts disabled.
1855 */
1856static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1857{
1858 AssertPtr(pVCpu);
1859 AssertPtr(pCpu);
1860 NOREF(pVM);
1861
1862 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1863
1864 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1865#if 0
1866 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1867 pVCpu->hm.s.TlbShootdown.cPages = 0;
1868#endif
1869
1870 Assert(pCpu->idCpu != NIL_RTCPUID);
1871 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1872 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1873 pVCpu->hm.s.fForceTLBFlush = false;
1874 return;
1875}
1876
1877
1878/**
1879 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1880 *
1881 * @param pVM Pointer to the VM.
1882 * @param pVCpu Pointer to the VMCPU.
1883 * @param pCpu Pointer to the global HM CPU struct.
1884 * @remarks All references to "ASID" in this function pertains to "VPID" in
1885 * Intel's nomenclature. The reason is, to avoid confusion in compare
1886 * statements since the host-CPU copies are named "ASID".
1887 *
1888 * @remarks Called with interrupts disabled.
1889 */
1890static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1891{
1892#ifdef VBOX_WITH_STATISTICS
1893 bool fTlbFlushed = false;
1894# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1895# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1896 if (!fTlbFlushed) \
1897 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1898 } while (0)
1899#else
1900# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1901# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1902#endif
1903
1904 AssertPtr(pVM);
1905 AssertPtr(pCpu);
1906 AssertPtr(pVCpu);
1907 Assert(pCpu->idCpu != NIL_RTCPUID);
1908
1909 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1910 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1911 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1912
1913 /*
1914 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1915 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1916 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1917 */
1918 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1919 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1920 {
1921 ++pCpu->uCurrentAsid;
1922 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1923 {
1924 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1925 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1926 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1927 }
1928
1929 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1930 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1931 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1932
1933 /*
1934 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1935 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1936 */
1937 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1938 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1939 HMVMX_SET_TAGGED_TLB_FLUSHED();
1940 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1941 }
1942
1943 /* Check for explicit TLB shootdowns. */
1944 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1945 {
1946 /*
1947 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1948 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1949 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1950 * but not guest-physical mappings.
1951 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1952 */
1953 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1955 HMVMX_SET_TAGGED_TLB_FLUSHED();
1956 }
1957
1958 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1959 * where it is commented out. Support individual entry flushing
1960 * someday. */
1961#if 0
1962 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1963 {
1964 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965
1966 /*
1967 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1968 * as supported by the CPU.
1969 */
1970 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1971 {
1972 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1973 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1974 }
1975 else
1976 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1977
1978 HMVMX_SET_TAGGED_TLB_FLUSHED();
1979 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1980 pVCpu->hm.s.TlbShootdown.cPages = 0;
1981 }
1982#endif
1983
1984 pVCpu->hm.s.fForceTLBFlush = false;
1985
1986 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1987
1988 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1989 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1990 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1991 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1992 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1993 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1994 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1995 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1996 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1997
1998 /* Update VMCS with the VPID. */
1999 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2000 AssertRC(rc);
2001
2002#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2003}
2004
2005
2006/**
2007 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2008 *
2009 * @returns VBox status code.
2010 * @param pVM Pointer to the VM.
2011 * @param pVCpu Pointer to the VMCPU.
2012 * @param pCpu Pointer to the global HM CPU struct.
2013 *
2014 * @remarks Called with interrupts disabled.
2015 */
2016static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2017{
2018 AssertPtr(pVM);
2019 AssertPtr(pVCpu);
2020 AssertPtr(pCpu);
2021 Assert(pCpu->idCpu != NIL_RTCPUID);
2022 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2023 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2024
2025 /*
2026 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2027 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2028 */
2029 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2030 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2031 {
2032 pVCpu->hm.s.fForceTLBFlush = true;
2033 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034 }
2035
2036 /* Check for explicit TLB shootdown flushes. */
2037 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2038 {
2039 pVCpu->hm.s.fForceTLBFlush = true;
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2041 }
2042
2043 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2044 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2045
2046 if (pVCpu->hm.s.fForceTLBFlush)
2047 {
2048 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2049 pVCpu->hm.s.fForceTLBFlush = false;
2050 }
2051 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2052 * where it is commented out. Support individual entry flushing
2053 * someday. */
2054#if 0
2055 else
2056 {
2057 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2058 {
2059 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2060 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2061 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2062 }
2063 else
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2065
2066 pVCpu->hm.s.TlbShootdown.cPages = 0;
2067 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2068 }
2069#endif
2070}
2071
2072
2073/**
2074 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2075 *
2076 * @returns VBox status code.
2077 * @param pVM Pointer to the VM.
2078 * @param pVCpu Pointer to the VMCPU.
2079 * @param pCpu Pointer to the global HM CPU struct.
2080 *
2081 * @remarks Called with interrupts disabled.
2082 */
2083static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2084{
2085 AssertPtr(pVM);
2086 AssertPtr(pVCpu);
2087 AssertPtr(pCpu);
2088 Assert(pCpu->idCpu != NIL_RTCPUID);
2089 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2090 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2091
2092 /*
2093 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2094 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2095 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2096 */
2097 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2098 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2099 {
2100 pVCpu->hm.s.fForceTLBFlush = true;
2101 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2102 }
2103
2104 /* Check for explicit TLB shootdown flushes. */
2105 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2106 {
2107 /*
2108 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2109 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2110 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2111 */
2112 pVCpu->hm.s.fForceTLBFlush = true;
2113 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2114 }
2115
2116 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2117 if (pVCpu->hm.s.fForceTLBFlush)
2118 {
2119 ++pCpu->uCurrentAsid;
2120 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2121 {
2122 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2123 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2124 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2125 }
2126
2127 pVCpu->hm.s.fForceTLBFlush = false;
2128 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2129 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2130 if (pCpu->fFlushAsidBeforeUse)
2131 {
2132 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2133 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2134 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2135 {
2136 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2137 pCpu->fFlushAsidBeforeUse = false;
2138 }
2139 else
2140 {
2141 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2142 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2143 }
2144 }
2145 }
2146 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2147 * where it is commented out. Support individual entry flushing
2148 * someday. */
2149#if 0
2150 else
2151 {
2152 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2153 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2154 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2155 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2156
2157 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2158 {
2159 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2160 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2161 {
2162 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2163 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2164 }
2165 else
2166 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2167
2168 pVCpu->hm.s.TlbShootdown.cPages = 0;
2169 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2170 }
2171 else
2172 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2173 }
2174#endif
2175
2176 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2177 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2178 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2179 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2180 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2181 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2182 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2183
2184 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2185 AssertRC(rc);
2186}
2187
2188
2189/**
2190 * Flushes the guest TLB entry based on CPU capabilities.
2191 *
2192 * @param pVCpu Pointer to the VMCPU.
2193 * @param pCpu Pointer to the global HM CPU struct.
2194 */
2195DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2196{
2197#ifdef HMVMX_ALWAYS_FLUSH_TLB
2198 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2199#endif
2200 PVM pVM = pVCpu->CTX_SUFF(pVM);
2201 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2202 {
2203 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2204 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2205 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2206 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2207 default:
2208 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2209 break;
2210 }
2211
2212 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2213 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2214
2215 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2216}
2217
2218
2219/**
2220 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2221 * TLB entries from the host TLB before VM-entry.
2222 *
2223 * @returns VBox status code.
2224 * @param pVM Pointer to the VM.
2225 */
2226static int hmR0VmxSetupTaggedTlb(PVM pVM)
2227{
2228 /*
2229 * Determine optimal flush type for Nested Paging.
2230 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2231 * guest execution (see hmR3InitFinalizeR0()).
2232 */
2233 if (pVM->hm.s.fNestedPaging)
2234 {
2235 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2236 {
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2238 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2239 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2240 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2241 else
2242 {
2243 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2244 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2245 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2246 }
2247
2248 /* Make sure the write-back cacheable memory type for EPT is supported. */
2249 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2250 {
2251 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2252 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2253 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2254 }
2255 }
2256 else
2257 {
2258 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2259 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2261 }
2262 }
2263
2264 /*
2265 * Determine optimal flush type for VPID.
2266 */
2267 if (pVM->hm.s.vmx.fVpid)
2268 {
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2270 {
2271 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2272 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2273 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2274 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2275 else
2276 {
2277 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2278 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2279 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2280 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2281 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2282 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2283 pVM->hm.s.vmx.fVpid = false;
2284 }
2285 }
2286 else
2287 {
2288 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2289 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2290 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2291 pVM->hm.s.vmx.fVpid = false;
2292 }
2293 }
2294
2295 /*
2296 * Setup the handler for flushing tagged-TLBs.
2297 */
2298 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2299 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2300 else if (pVM->hm.s.fNestedPaging)
2301 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2302 else if (pVM->hm.s.vmx.fVpid)
2303 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2304 else
2305 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2306 return VINF_SUCCESS;
2307}
2308
2309
2310/**
2311 * Sets up pin-based VM-execution controls in the VMCS.
2312 *
2313 * @returns VBox status code.
2314 * @param pVM Pointer to the VM.
2315 * @param pVCpu Pointer to the VMCPU.
2316 */
2317static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2318{
2319 AssertPtr(pVM);
2320 AssertPtr(pVCpu);
2321
2322 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2323 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2324
2325 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2326 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2327
2328 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2329 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2330
2331 /* Enable the VMX preemption timer. */
2332 if (pVM->hm.s.vmx.fUsePreemptTimer)
2333 {
2334 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2335 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2336 }
2337
2338 if ((val & zap) != val)
2339 {
2340 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2341 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2342 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2343 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2344 }
2345
2346 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2347 AssertRCReturn(rc, rc);
2348
2349 /* Update VCPU with the currently set pin-based VM-execution controls. */
2350 pVCpu->hm.s.vmx.u32PinCtls = val;
2351 return rc;
2352}
2353
2354
2355/**
2356 * Sets up processor-based VM-execution controls in the VMCS.
2357 *
2358 * @returns VBox status code.
2359 * @param pVM Pointer to the VM.
2360 * @param pVMCPU Pointer to the VMCPU.
2361 */
2362static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2363{
2364 AssertPtr(pVM);
2365 AssertPtr(pVCpu);
2366
2367 int rc = VERR_INTERNAL_ERROR_5;
2368 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2378
2379 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2380 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2381 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2382 {
2383 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2384 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2385 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2386 }
2387
2388 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2389 if (!pVM->hm.s.fNestedPaging)
2390 {
2391 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2392 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2393 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2394 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2395 }
2396
2397 /* Use TPR shadowing if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2399 {
2400 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2401 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2402 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2403 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2404 AssertRCReturn(rc, rc);
2405
2406 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2407 /* CR8 writes cause a VM-exit based on TPR threshold. */
2408 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2409 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2410 }
2411 else
2412 {
2413 /*
2414 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2415 * Set this control only for 64-bit guests.
2416 */
2417 if (pVM->hm.s.fAllow64BitGuests)
2418 {
2419 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2421 }
2422 }
2423
2424 /* Use MSR-bitmaps if supported by the CPU. */
2425 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2426 {
2427 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2428
2429 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2431 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2432 AssertRCReturn(rc, rc);
2433
2434 /*
2435 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2436 * automatically using dedicated fields in the VMCS.
2437 */
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443
2444#if HC_ARCH_BITS == 64
2445 /*
2446 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2447 */
2448 if (pVM->hm.s.fAllow64BitGuests)
2449 {
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454 }
2455#endif
2456 }
2457
2458 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2459 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2460 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2461
2462 if ((val & zap) != val)
2463 {
2464 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2465 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2466 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2467 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2468 }
2469
2470 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2471 AssertRCReturn(rc, rc);
2472
2473 /* Update VCPU with the currently set processor-based VM-execution controls. */
2474 pVCpu->hm.s.vmx.u32ProcCtls = val;
2475
2476 /*
2477 * Secondary processor-based VM-execution controls.
2478 */
2479 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2480 {
2481 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2482 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2483
2484 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2485 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2486
2487 if (pVM->hm.s.fNestedPaging)
2488 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2489 else
2490 {
2491 /*
2492 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2493 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2494 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2495 */
2496 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2497 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2498 }
2499
2500 if (pVM->hm.s.vmx.fVpid)
2501 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2502
2503 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2505
2506 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2507 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2508 * done dynamically. */
2509 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2510 {
2511 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2512 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2513 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2514 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2515 AssertRCReturn(rc, rc);
2516 }
2517
2518 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2519 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2520
2521 if ((val & zap) != val)
2522 {
2523 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2524 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2525 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2526 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2527 }
2528
2529 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2530 AssertRCReturn(rc, rc);
2531
2532 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2533 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2534 }
2535 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2536 {
2537 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2538 "available\n"));
2539 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2540 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2541 }
2542
2543 return VINF_SUCCESS;
2544}
2545
2546
2547/**
2548 * Sets up miscellaneous (everything other than Pin & Processor-based
2549 * VM-execution) control fields in the VMCS.
2550 *
2551 * @returns VBox status code.
2552 * @param pVM Pointer to the VM.
2553 * @param pVCpu Pointer to the VMCPU.
2554 */
2555static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2556{
2557 NOREF(pVM);
2558 AssertPtr(pVM);
2559 AssertPtr(pVCpu);
2560
2561 int rc = VERR_GENERAL_FAILURE;
2562
2563 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2564#if 0
2565 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2566 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2567 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2568
2569 /*
2570 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2571 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2572 * We thus use the exception bitmap to control it rather than use both.
2573 */
2574 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2575 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2576
2577 /** @todo Explore possibility of using IO-bitmaps. */
2578 /* All IO & IOIO instructions cause VM-exits. */
2579 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2580 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2581
2582 /* Initialize the MSR-bitmap area. */
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2584 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2585 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2586#endif
2587
2588 /* Setup MSR auto-load/store area. */
2589 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2590 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2591 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2592 AssertRCReturn(rc, rc);
2593 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2594 AssertRCReturn(rc, rc);
2595
2596 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2597 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2598 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2599 AssertRCReturn(rc, rc);
2600
2601 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2602 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2603 AssertRCReturn(rc, rc);
2604
2605 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2606#if 0
2607 /* Setup debug controls */
2608 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2609 AssertRCReturn(rc, rc);
2610 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2611 AssertRCReturn(rc, rc);
2612#endif
2613
2614 return rc;
2615}
2616
2617
2618/**
2619 * Sets up the initial exception bitmap in the VMCS based on static conditions
2620 * (i.e. conditions that cannot ever change after starting the VM).
2621 *
2622 * @returns VBox status code.
2623 * @param pVM Pointer to the VM.
2624 * @param pVCpu Pointer to the VMCPU.
2625 */
2626static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2627{
2628 AssertPtr(pVM);
2629 AssertPtr(pVCpu);
2630
2631 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2632
2633 uint32_t u32XcptBitmap = 0;
2634
2635 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2636 if (!pVM->hm.s.fNestedPaging)
2637 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2638
2639 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2640 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2641 AssertRCReturn(rc, rc);
2642 return rc;
2643}
2644
2645
2646/**
2647 * Sets up the initial guest-state mask. The guest-state mask is consulted
2648 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2649 * for the nested virtualization case (as it would cause a VM-exit).
2650 *
2651 * @param pVCpu Pointer to the VMCPU.
2652 */
2653static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2654{
2655 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2656 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2657 return VINF_SUCCESS;
2658}
2659
2660
2661/**
2662 * Does per-VM VT-x initialization.
2663 *
2664 * @returns VBox status code.
2665 * @param pVM Pointer to the VM.
2666 */
2667VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2668{
2669 LogFlowFunc(("pVM=%p\n", pVM));
2670
2671 int rc = hmR0VmxStructsAlloc(pVM);
2672 if (RT_FAILURE(rc))
2673 {
2674 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2675 return rc;
2676 }
2677
2678 return VINF_SUCCESS;
2679}
2680
2681
2682/**
2683 * Does per-VM VT-x termination.
2684 *
2685 * @returns VBox status code.
2686 * @param pVM Pointer to the VM.
2687 */
2688VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2689{
2690 LogFlowFunc(("pVM=%p\n", pVM));
2691
2692#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2693 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2694 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2695#endif
2696 hmR0VmxStructsFree(pVM);
2697 return VINF_SUCCESS;
2698}
2699
2700
2701/**
2702 * Sets up the VM for execution under VT-x.
2703 * This function is only called once per-VM during initialization.
2704 *
2705 * @returns VBox status code.
2706 * @param pVM Pointer to the VM.
2707 */
2708VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2709{
2710 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2711 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2712
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715 /*
2716 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2717 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2718 */
2719 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2720 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2721 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2722 || !pVM->hm.s.vmx.pRealModeTSS))
2723 {
2724 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2725 return VERR_INTERNAL_ERROR;
2726 }
2727
2728#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2729 /*
2730 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2731 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2732 */
2733 if ( pVM->hm.s.fAllow64BitGuests
2734 && !HMVMX_IS_64BIT_HOST_MODE())
2735 {
2736 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2737 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2738 }
2739#endif
2740
2741 /* Initialize these always, see hmR3InitFinalizeR0().*/
2742 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2743 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2744
2745 /* Setup the tagged-TLB flush handlers. */
2746 int rc = hmR0VmxSetupTaggedTlb(pVM);
2747 if (RT_FAILURE(rc))
2748 {
2749 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2750 return rc;
2751 }
2752
2753 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2754 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2755#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2756 if ( HMVMX_IS_64BIT_HOST_MODE()
2757 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2758 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2759 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2760 {
2761 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2762 }
2763#endif
2764
2765 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2766 {
2767 PVMCPU pVCpu = &pVM->aCpus[i];
2768 AssertPtr(pVCpu);
2769 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2770
2771 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2772 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2773
2774 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2775 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2776 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2777
2778 /* Set revision dword at the beginning of the VMCS structure. */
2779 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2780
2781 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2782 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2783 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2784 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2785
2786 /* Load this VMCS as the current VMCS. */
2787 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2788 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2789 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2790
2791 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2793 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2794
2795 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2801 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2802
2803 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2805 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2806
2807 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2809 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2810
2811#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2812 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815#endif
2816
2817 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2818 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2820 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2821
2822 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2823
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2825 }
2826
2827 return VINF_SUCCESS;
2828}
2829
2830
2831/**
2832 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2833 * the VMCS.
2834 *
2835 * @returns VBox status code.
2836 * @param pVM Pointer to the VM.
2837 * @param pVCpu Pointer to the VMCPU.
2838 */
2839DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2840{
2841 NOREF(pVM); NOREF(pVCpu);
2842
2843 RTCCUINTREG uReg = ASMGetCR0();
2844 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2845 AssertRCReturn(rc, rc);
2846
2847#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2848 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2849 if (HMVMX_IS_64BIT_HOST_MODE())
2850 {
2851 uint64_t uRegCR3 = HMR0Get64bitCR3();
2852 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2853 }
2854 else
2855#endif
2856 {
2857 uReg = ASMGetCR3();
2858 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2859 }
2860 AssertRCReturn(rc, rc);
2861
2862 uReg = ASMGetCR4();
2863 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2864 AssertRCReturn(rc, rc);
2865 return rc;
2866}
2867
2868
2869#if HC_ARCH_BITS == 64
2870/**
2871 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2872 * requirements. See hmR0VmxSaveHostSegmentRegs().
2873 */
2874# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2875 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2876 { \
2877 bool fValidSelector = true; \
2878 if ((selValue) & X86_SEL_LDT) \
2879 { \
2880 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2881 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2882 } \
2883 if (fValidSelector) \
2884 { \
2885 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2886 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2887 } \
2888 (selValue) = 0; \
2889 }
2890#endif
2891
2892
2893/**
2894 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2895 * the host-state area in the VMCS.
2896 *
2897 * @returns VBox status code.
2898 * @param pVM Pointer to the VM.
2899 * @param pVCpu Pointer to the VMCPU.
2900 */
2901DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2902{
2903 int rc = VERR_INTERNAL_ERROR_5;
2904
2905#if HC_ARCH_BITS == 64
2906 /*
2907 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2908 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2909 */
2910 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2911 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2912#endif
2913
2914 /*
2915 * Host DS, ES, FS and GS segment registers.
2916 */
2917#if HC_ARCH_BITS == 64
2918 RTSEL uSelDS = ASMGetDS();
2919 RTSEL uSelES = ASMGetES();
2920 RTSEL uSelFS = ASMGetFS();
2921 RTSEL uSelGS = ASMGetGS();
2922#else
2923 RTSEL uSelDS = 0;
2924 RTSEL uSelES = 0;
2925 RTSEL uSelFS = 0;
2926 RTSEL uSelGS = 0;
2927#endif
2928
2929 /* Recalculate which host-state bits need to be manually restored. */
2930 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2931
2932 /*
2933 * Host CS and SS segment registers.
2934 */
2935#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2936 RTSEL uSelCS;
2937 RTSEL uSelSS;
2938 if (HMVMX_IS_64BIT_HOST_MODE())
2939 {
2940 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2941 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2942 }
2943 else
2944 {
2945 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2946 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2947 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2948 }
2949#else
2950 RTSEL uSelCS = ASMGetCS();
2951 RTSEL uSelSS = ASMGetSS();
2952#endif
2953
2954 /*
2955 * Host TR segment register.
2956 */
2957 RTSEL uSelTR = ASMGetTR();
2958
2959#if HC_ARCH_BITS == 64
2960 /*
2961 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2962 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2963 */
2964 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2965 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2966 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2967 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2968# undef VMXLOCAL_ADJUST_HOST_SEG
2969#endif
2970
2971 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2972 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2973 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2974 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2975 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2976 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2977 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2978 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2979 Assert(uSelCS);
2980 Assert(uSelTR);
2981
2982 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2983#if 0
2984 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2985 Assert(uSelSS != 0);
2986#endif
2987
2988 /* Write these host selector fields into the host-state area in the VMCS. */
2989 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2990 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2991#if HC_ARCH_BITS == 64
2992 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2994 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2996#endif
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2998
2999 /*
3000 * Host GDTR and IDTR.
3001 */
3002 RTGDTR Gdtr;
3003 RT_ZERO(Gdtr);
3004#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3005 if (HMVMX_IS_64BIT_HOST_MODE())
3006 {
3007 X86XDTR64 Gdtr64;
3008 X86XDTR64 Idtr64;
3009 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3010 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3011 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3012
3013 Gdtr.cbGdt = Gdtr64.cb;
3014 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3015 }
3016 else
3017#endif
3018 {
3019 RTIDTR Idtr;
3020 ASMGetGDTR(&Gdtr);
3021 ASMGetIDTR(&Idtr);
3022 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3023 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3024
3025#if HC_ARCH_BITS == 64
3026 /*
3027 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3028 * maximum limit (0xffff) on every VM-exit.
3029 */
3030 if (Gdtr.cbGdt != 0xffff)
3031 {
3032 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3033 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3034 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3035 }
3036
3037 /*
3038 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3039 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3040 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3041 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3042 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3043 * hosts where we are pretty sure it won't cause trouble.
3044 */
3045# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3046 if (Idtr.cbIdt < 0x0fff)
3047# else
3048 if (Idtr.cbIdt != 0xffff)
3049# endif
3050 {
3051 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3052 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3053 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3054 }
3055#endif
3056 }
3057
3058 /*
3059 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3060 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3061 */
3062 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3063 {
3064 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3065 return VERR_VMX_INVALID_HOST_STATE;
3066 }
3067
3068 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3069#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3070 if (HMVMX_IS_64BIT_HOST_MODE())
3071 {
3072 /* We need the 64-bit TR base for hybrid darwin. */
3073 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3074 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3075 }
3076 else
3077#endif
3078 {
3079 uintptr_t uTRBase;
3080#if HC_ARCH_BITS == 64
3081 uTRBase = X86DESC64_BASE(pDesc);
3082
3083 /*
3084 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3085 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3086 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3087 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3088 *
3089 * [1] See Intel spec. 3.5 "System Descriptor Types".
3090 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3091 */
3092 Assert(pDesc->System.u4Type == 11);
3093 if ( pDesc->System.u16LimitLow != 0x67
3094 || pDesc->System.u4LimitHigh)
3095 {
3096 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3097 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3098 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3099 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3100 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3101
3102 /* Store the GDTR here as we need it while restoring TR. */
3103 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3104 }
3105#else
3106 uTRBase = X86DESC_BASE(pDesc);
3107#endif
3108 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3109 }
3110 AssertRCReturn(rc, rc);
3111
3112 /*
3113 * Host FS base and GS base.
3114 */
3115#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3116 if (HMVMX_IS_64BIT_HOST_MODE())
3117 {
3118 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3119 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3120 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3121 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3122
3123# if HC_ARCH_BITS == 64
3124 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3125 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3126 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3127 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3128 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3129# endif
3130 }
3131#endif
3132 return rc;
3133}
3134
3135
3136/**
3137 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3138 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3139 * the host after every successful VM-exit.
3140 *
3141 * @returns VBox status code.
3142 * @param pVM Pointer to the VM.
3143 * @param pVCpu Pointer to the VMCPU.
3144 *
3145 * @remarks No-long-jump zone!!!
3146 */
3147DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3148{
3149 NOREF(pVM);
3150
3151 AssertPtr(pVCpu);
3152 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3153
3154 int rc = VINF_SUCCESS;
3155#if HC_ARCH_BITS == 64
3156 if (pVM->hm.s.fAllow64BitGuests)
3157 hmR0VmxLazySaveHostMsrs(pVCpu);
3158#endif
3159
3160 /*
3161 * Host Sysenter MSRs.
3162 */
3163 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3164 AssertRCReturn(rc, rc);
3165#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3166 if (HMVMX_IS_64BIT_HOST_MODE())
3167 {
3168 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3169 AssertRCReturn(rc, rc);
3170 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3171 }
3172 else
3173 {
3174 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3175 AssertRCReturn(rc, rc);
3176 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3177 }
3178#elif HC_ARCH_BITS == 32
3179 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3180 AssertRCReturn(rc, rc);
3181 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3182#else
3183 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3184 AssertRCReturn(rc, rc);
3185 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3186#endif
3187 AssertRCReturn(rc, rc);
3188
3189 /*
3190 * Host EFER MSR.
3191 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3192 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3193 */
3194 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3195 {
3196 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3197 AssertRCReturn(rc, rc);
3198 }
3199
3200 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3201 * hmR0VmxLoadGuestExitCtls() !! */
3202
3203 return rc;
3204}
3205
3206
3207/**
3208 * Figures out if we need to swap the EFER MSR which is
3209 * particularly expensive.
3210 *
3211 * We check all relevant bits. For now, that's everything
3212 * besides LMA/LME, as these two bits are handled by VM-entry,
3213 * see hmR0VmxLoadGuestExitCtls() and
3214 * hmR0VMxLoadGuestEntryCtls().
3215 *
3216 * @returns true if we need to load guest EFER, false otherwise.
3217 * @param pVCpu Pointer to the VMCPU.
3218 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3219 * out-of-sync. Make sure to update the required fields
3220 * before using them.
3221 *
3222 * @remarks Requires EFER, CR4.
3223 * @remarks No-long-jump zone!!!
3224 */
3225static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3226{
3227#ifdef HMVMX_ALWAYS_SWAP_EFER
3228 return true;
3229#endif
3230
3231#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3232 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3233 if (CPUMIsGuestInLongMode(pVCpu))
3234 return false;
3235#endif
3236
3237 PVM pVM = pVCpu->CTX_SUFF(pVM);
3238 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3239 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3240
3241 /*
3242 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3243 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3244 */
3245 if ( CPUMIsGuestInLongMode(pVCpu)
3246 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3247 {
3248 return true;
3249 }
3250
3251 /*
3252 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3253 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3254 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3255 */
3256 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3257 && (pMixedCtx->cr0 & X86_CR0_PG)
3258 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3259 {
3260 /* Assert that host is PAE capable. */
3261 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3262 return true;
3263 }
3264
3265 /** @todo Check the latest Intel spec. for any other bits,
3266 * like SMEP/SMAP? */
3267 return false;
3268}
3269
3270
3271/**
3272 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3273 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3274 * controls".
3275 *
3276 * @returns VBox status code.
3277 * @param pVCpu Pointer to the VMCPU.
3278 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3279 * out-of-sync. Make sure to update the required fields
3280 * before using them.
3281 *
3282 * @remarks Requires EFER.
3283 * @remarks No-long-jump zone!!!
3284 */
3285DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3286{
3287 int rc = VINF_SUCCESS;
3288 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3289 {
3290 PVM pVM = pVCpu->CTX_SUFF(pVM);
3291 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3292 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3293
3294 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3295 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3296
3297 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3298 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3299 {
3300 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3301 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3302 }
3303 else
3304 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3305
3306 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3307 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3308 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3309 {
3310 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3311 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3312 }
3313
3314 /*
3315 * The following should -not- be set (since we're not in SMM mode):
3316 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3317 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3318 */
3319
3320 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3321 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3322
3323 if ((val & zap) != val)
3324 {
3325 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3326 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3327 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3328 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3329 }
3330
3331 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3332 AssertRCReturn(rc, rc);
3333
3334 /* Update VCPU with the currently set VM-exit controls. */
3335 pVCpu->hm.s.vmx.u32EntryCtls = val;
3336 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3337 }
3338 return rc;
3339}
3340
3341
3342/**
3343 * Sets up the VM-exit controls in the VMCS.
3344 *
3345 * @returns VBox status code.
3346 * @param pVM Pointer to the VM.
3347 * @param pVCpu Pointer to the VMCPU.
3348 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3349 * out-of-sync. Make sure to update the required fields
3350 * before using them.
3351 *
3352 * @remarks Requires EFER.
3353 */
3354DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3355{
3356 NOREF(pMixedCtx);
3357
3358 int rc = VINF_SUCCESS;
3359 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3360 {
3361 PVM pVM = pVCpu->CTX_SUFF(pVM);
3362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3364
3365 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3366 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3367
3368 /*
3369 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3370 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3371 */
3372#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3373 if (HMVMX_IS_64BIT_HOST_MODE())
3374 {
3375 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3376 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3377 }
3378 else
3379 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3380#else
3381 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3382 {
3383 /* The switcher returns to long mode, EFER is managed by the switcher. */
3384 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3385 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3386 }
3387 else
3388 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3389#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3390
3391 /* If the newer VMCS fields for managing EFER exists, use it. */
3392 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3393 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3394 {
3395 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3396 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3397 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3398 }
3399
3400 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3401 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3402
3403 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3404 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3405 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3406
3407 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3408 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3409
3410 if ((val & zap) != val)
3411 {
3412 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3413 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3414 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3415 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3416 }
3417
3418 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3419 AssertRCReturn(rc, rc);
3420
3421 /* Update VCPU with the currently set VM-exit controls. */
3422 pVCpu->hm.s.vmx.u32ExitCtls = val;
3423 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3424 }
3425 return rc;
3426}
3427
3428
3429/**
3430 * Loads the guest APIC and related state.
3431 *
3432 * @returns VBox status code.
3433 * @param pVM Pointer to the VM.
3434 * @param pVCpu Pointer to the VMCPU.
3435 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3436 * out-of-sync. Make sure to update the required fields
3437 * before using them.
3438 */
3439DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3440{
3441 NOREF(pMixedCtx);
3442
3443 int rc = VINF_SUCCESS;
3444 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3445 {
3446 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3447 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3448 {
3449 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3450
3451 bool fPendingIntr = false;
3452 uint8_t u8Tpr = 0;
3453 uint8_t u8PendingIntr = 0;
3454 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3455 AssertRCReturn(rc, rc);
3456
3457 /*
3458 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3459 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3460 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3461 * the interrupt when we VM-exit for other reasons.
3462 */
3463 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3464 uint32_t u32TprThreshold = 0;
3465 if (fPendingIntr)
3466 {
3467 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3468 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3469 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3470 if (u8PendingPriority <= u8TprPriority)
3471 u32TprThreshold = u8PendingPriority;
3472 else
3473 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3474 }
3475 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3476
3477 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3478 AssertRCReturn(rc, rc);
3479 }
3480
3481 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3482 }
3483 return rc;
3484}
3485
3486
3487/**
3488 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3489 *
3490 * @returns Guest's interruptibility-state.
3491 * @param pVCpu Pointer to the VMCPU.
3492 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3493 * out-of-sync. Make sure to update the required fields
3494 * before using them.
3495 *
3496 * @remarks No-long-jump zone!!!
3497 */
3498DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3499{
3500 /*
3501 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3502 */
3503 uint32_t uIntrState = 0;
3504 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3505 {
3506 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3507 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3508 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3509 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3510 {
3511 if (pMixedCtx->eflags.Bits.u1IF)
3512 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3513 else
3514 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3515 }
3516 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3517 }
3518
3519 /*
3520 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3521 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3522 * setting this would block host-NMIs and IRET will not clear the blocking.
3523 *
3524 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3525 */
3526 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3527 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3528 {
3529 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3530 }
3531
3532 return uIntrState;
3533}
3534
3535
3536/**
3537 * Loads the guest's interruptibility-state into the guest-state area in the
3538 * VMCS.
3539 *
3540 * @returns VBox status code.
3541 * @param pVCpu Pointer to the VMCPU.
3542 * @param uIntrState The interruptibility-state to set.
3543 */
3544static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3545{
3546 NOREF(pVCpu);
3547 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3548 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3549 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3550 AssertRCReturn(rc, rc);
3551 return rc;
3552}
3553
3554
3555/**
3556 * Loads the guest's RIP into the guest-state area in the VMCS.
3557 *
3558 * @returns VBox status code.
3559 * @param pVCpu Pointer to the VMCPU.
3560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3561 * out-of-sync. Make sure to update the required fields
3562 * before using them.
3563 *
3564 * @remarks No-long-jump zone!!!
3565 */
3566static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3567{
3568 int rc = VINF_SUCCESS;
3569 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3570 {
3571 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3572 AssertRCReturn(rc, rc);
3573
3574 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3575 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3576 HMCPU_CF_VALUE(pVCpu)));
3577 }
3578 return rc;
3579}
3580
3581
3582/**
3583 * Loads the guest's RSP into the guest-state area in the VMCS.
3584 *
3585 * @returns VBox status code.
3586 * @param pVCpu Pointer to the VMCPU.
3587 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3588 * out-of-sync. Make sure to update the required fields
3589 * before using them.
3590 *
3591 * @remarks No-long-jump zone!!!
3592 */
3593static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3594{
3595 int rc = VINF_SUCCESS;
3596 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3597 {
3598 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3599 AssertRCReturn(rc, rc);
3600
3601 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3602 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3603 }
3604 return rc;
3605}
3606
3607
3608/**
3609 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3610 *
3611 * @returns VBox status code.
3612 * @param pVCpu Pointer to the VMCPU.
3613 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3614 * out-of-sync. Make sure to update the required fields
3615 * before using them.
3616 *
3617 * @remarks No-long-jump zone!!!
3618 */
3619static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3620{
3621 int rc = VINF_SUCCESS;
3622 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3623 {
3624 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3625 Let us assert it as such and use 32-bit VMWRITE. */
3626 Assert(!(pMixedCtx->rflags.u64 >> 32));
3627 X86EFLAGS Eflags = pMixedCtx->eflags;
3628 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3629 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3630
3631 /*
3632 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3633 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3634 */
3635 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3636 {
3637 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3638 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3639 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3640 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3641 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3642 }
3643
3644 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3645 AssertRCReturn(rc, rc);
3646
3647 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3648 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3649 }
3650 return rc;
3651}
3652
3653
3654/**
3655 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3656 *
3657 * @returns VBox status code.
3658 * @param pVCpu Pointer to the VMCPU.
3659 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3660 * out-of-sync. Make sure to update the required fields
3661 * before using them.
3662 *
3663 * @remarks No-long-jump zone!!!
3664 */
3665DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3666{
3667 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3668 AssertRCReturn(rc, rc);
3669 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3670 AssertRCReturn(rc, rc);
3671 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3672 AssertRCReturn(rc, rc);
3673 return rc;
3674}
3675
3676
3677/**
3678 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3679 * CR0 is partially shared with the host and we have to consider the FPU bits.
3680 *
3681 * @returns VBox status code.
3682 * @param pVM Pointer to the VM.
3683 * @param pVCpu Pointer to the VMCPU.
3684 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3685 * out-of-sync. Make sure to update the required fields
3686 * before using them.
3687 *
3688 * @remarks No-long-jump zone!!!
3689 */
3690static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3691{
3692 /*
3693 * Guest CR0.
3694 * Guest FPU.
3695 */
3696 int rc = VINF_SUCCESS;
3697 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3698 {
3699 Assert(!(pMixedCtx->cr0 >> 32));
3700 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3701 PVM pVM = pVCpu->CTX_SUFF(pVM);
3702
3703 /* The guest's view (read access) of its CR0 is unblemished. */
3704 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3705 AssertRCReturn(rc, rc);
3706 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3707
3708 /* Setup VT-x's view of the guest CR0. */
3709 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3710 if (pVM->hm.s.fNestedPaging)
3711 {
3712 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3713 {
3714 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3715 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3716 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3717 }
3718 else
3719 {
3720 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3721 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3723 }
3724
3725 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3726 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3727 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3728
3729 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3730 AssertRCReturn(rc, rc);
3731 }
3732 else
3733 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3734
3735 /*
3736 * Guest FPU bits.
3737 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3738 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3739 */
3740 u32GuestCR0 |= X86_CR0_NE;
3741 bool fInterceptNM = false;
3742 if (CPUMIsGuestFPUStateActive(pVCpu))
3743 {
3744 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3745 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3746 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3747 }
3748 else
3749 {
3750 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3751 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3752 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3753 }
3754
3755 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3756 bool fInterceptMF = false;
3757 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3758 fInterceptMF = true;
3759
3760 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3761 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3762 {
3763 Assert(PDMVmmDevHeapIsEnabled(pVM));
3764 Assert(pVM->hm.s.vmx.pRealModeTSS);
3765 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3766 fInterceptNM = true;
3767 fInterceptMF = true;
3768 }
3769 else
3770 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3771
3772 if (fInterceptNM)
3773 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3774 else
3775 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3776
3777 if (fInterceptMF)
3778 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3779 else
3780 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3781
3782 /* Additional intercepts for debugging, define these yourself explicitly. */
3783#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3784 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3785 | RT_BIT(X86_XCPT_BP)
3786 | RT_BIT(X86_XCPT_DB)
3787 | RT_BIT(X86_XCPT_DE)
3788 | RT_BIT(X86_XCPT_NM)
3789 | RT_BIT(X86_XCPT_TS)
3790 | RT_BIT(X86_XCPT_UD)
3791 | RT_BIT(X86_XCPT_NP)
3792 | RT_BIT(X86_XCPT_SS)
3793 | RT_BIT(X86_XCPT_GP)
3794 | RT_BIT(X86_XCPT_PF)
3795 | RT_BIT(X86_XCPT_MF)
3796 ;
3797#elif defined(HMVMX_ALWAYS_TRAP_PF)
3798 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3799#endif
3800
3801 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3802
3803 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3804 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3805 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3806 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3807 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3808 else
3809 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3810
3811 u32GuestCR0 |= uSetCR0;
3812 u32GuestCR0 &= uZapCR0;
3813 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3814
3815 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3816 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3817 AssertRCReturn(rc, rc);
3818 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3819 AssertRCReturn(rc, rc);
3820 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3821 uZapCR0));
3822
3823 /*
3824 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3825 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3826 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3827 */
3828 uint32_t u32CR0Mask = 0;
3829 u32CR0Mask = X86_CR0_PE
3830 | X86_CR0_NE
3831 | X86_CR0_WP
3832 | X86_CR0_PG
3833 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3834 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3835 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3836
3837 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3838 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3839 * and @bugref{6944}. */
3840#if 0
3841 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3842 u32CR0Mask &= ~X86_CR0_PE;
3843#endif
3844 if (pVM->hm.s.fNestedPaging)
3845 u32CR0Mask &= ~X86_CR0_WP;
3846
3847 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3848 if (fInterceptNM)
3849 {
3850 u32CR0Mask |= X86_CR0_TS
3851 | X86_CR0_MP;
3852 }
3853
3854 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3855 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3856 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3857 AssertRCReturn(rc, rc);
3858 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3859
3860 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3861 }
3862 return rc;
3863}
3864
3865
3866/**
3867 * Loads the guest control registers (CR3, CR4) into the guest-state area
3868 * in the VMCS.
3869 *
3870 * @returns VBox status code.
3871 * @param pVM Pointer to the VM.
3872 * @param pVCpu Pointer to the VMCPU.
3873 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3874 * out-of-sync. Make sure to update the required fields
3875 * before using them.
3876 *
3877 * @remarks No-long-jump zone!!!
3878 */
3879static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3880{
3881 int rc = VINF_SUCCESS;
3882 PVM pVM = pVCpu->CTX_SUFF(pVM);
3883
3884 /*
3885 * Guest CR2.
3886 * It's always loaded in the assembler code. Nothing to do here.
3887 */
3888
3889 /*
3890 * Guest CR3.
3891 */
3892 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3893 {
3894 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3895 if (pVM->hm.s.fNestedPaging)
3896 {
3897 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3898
3899 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3900 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3901 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3902 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3903
3904 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3905 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3906 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3907
3908 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3909 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3910 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3911 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3912
3913 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3914 AssertRCReturn(rc, rc);
3915 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3916
3917 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3918 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3919 {
3920 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3921 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3922 {
3923 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3924 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3925 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3926 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3927 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3928 }
3929
3930 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3931 have Unrestricted Execution to handle the guest when it's not using paging. */
3932 GCPhysGuestCR3 = pMixedCtx->cr3;
3933 }
3934 else
3935 {
3936 /*
3937 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3938 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3939 * EPT takes care of translating it to host-physical addresses.
3940 */
3941 RTGCPHYS GCPhys;
3942 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3943 Assert(PDMVmmDevHeapIsEnabled(pVM));
3944
3945 /* We obtain it here every time as the guest could have relocated this PCI region. */
3946 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3947 AssertRCReturn(rc, rc);
3948
3949 GCPhysGuestCR3 = GCPhys;
3950 }
3951
3952 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3953 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3954 }
3955 else
3956 {
3957 /* Non-nested paging case, just use the hypervisor's CR3. */
3958 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3959
3960 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3961 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3962 }
3963 AssertRCReturn(rc, rc);
3964
3965 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3966 }
3967
3968 /*
3969 * Guest CR4.
3970 */
3971 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3972 {
3973 Assert(!(pMixedCtx->cr4 >> 32));
3974 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3975
3976 /* The guest's view of its CR4 is unblemished. */
3977 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3978 AssertRCReturn(rc, rc);
3979 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3980
3981 /* Setup VT-x's view of the guest CR4. */
3982 /*
3983 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3984 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3985 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3986 */
3987 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3988 {
3989 Assert(pVM->hm.s.vmx.pRealModeTSS);
3990 Assert(PDMVmmDevHeapIsEnabled(pVM));
3991 u32GuestCR4 &= ~X86_CR4_VME;
3992 }
3993
3994 if (pVM->hm.s.fNestedPaging)
3995 {
3996 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3997 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3998 {
3999 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4000 u32GuestCR4 |= X86_CR4_PSE;
4001 /* Our identity mapping is a 32-bit page directory. */
4002 u32GuestCR4 &= ~X86_CR4_PAE;
4003 }
4004 /* else use guest CR4.*/
4005 }
4006 else
4007 {
4008 /*
4009 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4010 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4011 */
4012 switch (pVCpu->hm.s.enmShadowMode)
4013 {
4014 case PGMMODE_REAL: /* Real-mode. */
4015 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4016 case PGMMODE_32_BIT: /* 32-bit paging. */
4017 {
4018 u32GuestCR4 &= ~X86_CR4_PAE;
4019 break;
4020 }
4021
4022 case PGMMODE_PAE: /* PAE paging. */
4023 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4024 {
4025 u32GuestCR4 |= X86_CR4_PAE;
4026 break;
4027 }
4028
4029 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4030 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4031#ifdef VBOX_ENABLE_64_BITS_GUESTS
4032 break;
4033#endif
4034 default:
4035 AssertFailed();
4036 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4037 }
4038 }
4039
4040 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4041 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4042 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4043 u32GuestCR4 |= uSetCR4;
4044 u32GuestCR4 &= uZapCR4;
4045
4046 /* Write VT-x's view of the guest CR4 into the VMCS. */
4047 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4048 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4049 AssertRCReturn(rc, rc);
4050
4051 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4052 uint32_t u32CR4Mask = 0;
4053 u32CR4Mask = X86_CR4_VME
4054 | X86_CR4_PAE
4055 | X86_CR4_PGE
4056 | X86_CR4_PSE
4057 | X86_CR4_VMXE;
4058 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4059 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4060 AssertRCReturn(rc, rc);
4061
4062 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4063 }
4064 return rc;
4065}
4066
4067
4068/**
4069 * Loads the guest debug registers into the guest-state area in the VMCS.
4070 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4071 *
4072 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4073 *
4074 * @returns VBox status code.
4075 * @param pVCpu Pointer to the VMCPU.
4076 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4077 * out-of-sync. Make sure to update the required fields
4078 * before using them.
4079 *
4080 * @remarks No-long-jump zone!!!
4081 */
4082static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4083{
4084 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4085 return VINF_SUCCESS;
4086
4087#ifdef VBOX_STRICT
4088 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4089 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4090 {
4091 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4092 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4093 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4094 }
4095#endif
4096
4097 int rc;
4098 PVM pVM = pVCpu->CTX_SUFF(pVM);
4099 bool fInterceptDB = false;
4100 bool fInterceptMovDRx = false;
4101 if ( pVCpu->hm.s.fSingleInstruction
4102 || DBGFIsStepping(pVCpu))
4103 {
4104 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4105 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4106 {
4107 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4108 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4109 AssertRCReturn(rc, rc);
4110 Assert(fInterceptDB == false);
4111 }
4112 else
4113 {
4114 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4115 pVCpu->hm.s.fClearTrapFlag = true;
4116 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4117 fInterceptDB = true;
4118 }
4119 }
4120
4121 if ( fInterceptDB
4122 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4123 {
4124 /*
4125 * Use the combined guest and host DRx values found in the hypervisor
4126 * register set because the debugger has breakpoints active or someone
4127 * is single stepping on the host side without a monitor trap flag.
4128 *
4129 * Note! DBGF expects a clean DR6 state before executing guest code.
4130 */
4131#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4132 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4133 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4134 {
4135 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4136 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4137 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4138 }
4139 else
4140#endif
4141 if (!CPUMIsHyperDebugStateActive(pVCpu))
4142 {
4143 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4144 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4145 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4146 }
4147
4148 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4149 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4150 AssertRCReturn(rc, rc);
4151
4152 pVCpu->hm.s.fUsingHyperDR7 = true;
4153 fInterceptDB = true;
4154 fInterceptMovDRx = true;
4155 }
4156 else
4157 {
4158 /*
4159 * If the guest has enabled debug registers, we need to load them prior to
4160 * executing guest code so they'll trigger at the right time.
4161 */
4162 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4163 {
4164#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4165 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4166 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4167 {
4168 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4169 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4170 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4171 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4172 }
4173 else
4174#endif
4175 if (!CPUMIsGuestDebugStateActive(pVCpu))
4176 {
4177 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4178 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4179 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4180 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4181 }
4182 Assert(!fInterceptDB);
4183 Assert(!fInterceptMovDRx);
4184 }
4185 /*
4186 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4187 * must intercept #DB in order to maintain a correct DR6 guest value.
4188 */
4189#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4190 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4191 && !CPUMIsGuestDebugStateActive(pVCpu))
4192#else
4193 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4194#endif
4195 {
4196 fInterceptMovDRx = true;
4197 fInterceptDB = true;
4198 }
4199
4200 /* Update guest DR7. */
4201 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4202 AssertRCReturn(rc, rc);
4203
4204 pVCpu->hm.s.fUsingHyperDR7 = false;
4205 }
4206
4207 /*
4208 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4209 */
4210 if (fInterceptDB)
4211 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4212 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4213 {
4214#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4215 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4216#endif
4217 }
4218 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4219 AssertRCReturn(rc, rc);
4220
4221 /*
4222 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4223 */
4224 if (fInterceptMovDRx)
4225 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4226 else
4227 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4228 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4229 AssertRCReturn(rc, rc);
4230
4231 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4232 return VINF_SUCCESS;
4233}
4234
4235
4236#ifdef VBOX_STRICT
4237/**
4238 * Strict function to validate segment registers.
4239 *
4240 * @remarks ASSUMES CR0 is up to date.
4241 */
4242static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4243{
4244 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4245 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4246 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4247 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4248 && ( !CPUMIsGuestInRealModeEx(pCtx)
4249 && !CPUMIsGuestInV86ModeEx(pCtx)))
4250 {
4251 /* Protected mode checks */
4252 /* CS */
4253 Assert(pCtx->cs.Attr.n.u1Present);
4254 Assert(!(pCtx->cs.Attr.u & 0xf00));
4255 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4256 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4257 || !(pCtx->cs.Attr.n.u1Granularity));
4258 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4259 || (pCtx->cs.Attr.n.u1Granularity));
4260 /* CS cannot be loaded with NULL in protected mode. */
4261 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4262 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4263 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4264 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4265 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4266 else
4267 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4268 /* SS */
4269 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4270 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4271 if ( !(pCtx->cr0 & X86_CR0_PE)
4272 || pCtx->cs.Attr.n.u4Type == 3)
4273 {
4274 Assert(!pCtx->ss.Attr.n.u2Dpl);
4275 }
4276 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4277 {
4278 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4279 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4280 Assert(pCtx->ss.Attr.n.u1Present);
4281 Assert(!(pCtx->ss.Attr.u & 0xf00));
4282 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4283 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4284 || !(pCtx->ss.Attr.n.u1Granularity));
4285 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4286 || (pCtx->ss.Attr.n.u1Granularity));
4287 }
4288 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4289 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4290 {
4291 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4292 Assert(pCtx->ds.Attr.n.u1Present);
4293 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4294 Assert(!(pCtx->ds.Attr.u & 0xf00));
4295 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4296 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4297 || !(pCtx->ds.Attr.n.u1Granularity));
4298 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4299 || (pCtx->ds.Attr.n.u1Granularity));
4300 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4301 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4302 }
4303 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4304 {
4305 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4306 Assert(pCtx->es.Attr.n.u1Present);
4307 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4308 Assert(!(pCtx->es.Attr.u & 0xf00));
4309 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4310 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4311 || !(pCtx->es.Attr.n.u1Granularity));
4312 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4313 || (pCtx->es.Attr.n.u1Granularity));
4314 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4315 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4316 }
4317 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4318 {
4319 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4320 Assert(pCtx->fs.Attr.n.u1Present);
4321 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4322 Assert(!(pCtx->fs.Attr.u & 0xf00));
4323 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4324 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4325 || !(pCtx->fs.Attr.n.u1Granularity));
4326 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4327 || (pCtx->fs.Attr.n.u1Granularity));
4328 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4329 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4330 }
4331 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4332 {
4333 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4334 Assert(pCtx->gs.Attr.n.u1Present);
4335 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4336 Assert(!(pCtx->gs.Attr.u & 0xf00));
4337 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4338 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4339 || !(pCtx->gs.Attr.n.u1Granularity));
4340 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4341 || (pCtx->gs.Attr.n.u1Granularity));
4342 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4343 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4344 }
4345 /* 64-bit capable CPUs. */
4346# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4347 if (HMVMX_IS_64BIT_HOST_MODE())
4348 {
4349 Assert(!(pCtx->cs.u64Base >> 32));
4350 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4351 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4352 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4353 }
4354# endif
4355 }
4356 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4357 || ( CPUMIsGuestInRealModeEx(pCtx)
4358 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4359 {
4360 /* Real and v86 mode checks. */
4361 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4362 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4363 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4364 {
4365 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4366 }
4367 else
4368 {
4369 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4370 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4371 }
4372
4373 /* CS */
4374 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4375 Assert(pCtx->cs.u32Limit == 0xffff);
4376 Assert(u32CSAttr == 0xf3);
4377 /* SS */
4378 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4379 Assert(pCtx->ss.u32Limit == 0xffff);
4380 Assert(u32SSAttr == 0xf3);
4381 /* DS */
4382 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4383 Assert(pCtx->ds.u32Limit == 0xffff);
4384 Assert(u32DSAttr == 0xf3);
4385 /* ES */
4386 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4387 Assert(pCtx->es.u32Limit == 0xffff);
4388 Assert(u32ESAttr == 0xf3);
4389 /* FS */
4390 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4391 Assert(pCtx->fs.u32Limit == 0xffff);
4392 Assert(u32FSAttr == 0xf3);
4393 /* GS */
4394 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4395 Assert(pCtx->gs.u32Limit == 0xffff);
4396 Assert(u32GSAttr == 0xf3);
4397 /* 64-bit capable CPUs. */
4398# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4399 if (HMVMX_IS_64BIT_HOST_MODE())
4400 {
4401 Assert(!(pCtx->cs.u64Base >> 32));
4402 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4403 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4404 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4405 }
4406# endif
4407 }
4408}
4409#endif /* VBOX_STRICT */
4410
4411
4412/**
4413 * Writes a guest segment register into the guest-state area in the VMCS.
4414 *
4415 * @returns VBox status code.
4416 * @param pVCpu Pointer to the VMCPU.
4417 * @param idxSel Index of the selector in the VMCS.
4418 * @param idxLimit Index of the segment limit in the VMCS.
4419 * @param idxBase Index of the segment base in the VMCS.
4420 * @param idxAccess Index of the access rights of the segment in the VMCS.
4421 * @param pSelReg Pointer to the segment selector.
4422 *
4423 * @remarks No-long-jump zone!!!
4424 */
4425static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4426 uint32_t idxAccess, PCPUMSELREG pSelReg)
4427{
4428 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4429 AssertRCReturn(rc, rc);
4430 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4431 AssertRCReturn(rc, rc);
4432 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4433 AssertRCReturn(rc, rc);
4434
4435 uint32_t u32Access = pSelReg->Attr.u;
4436 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4437 {
4438 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4439 u32Access = 0xf3;
4440 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4441 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4442 }
4443 else
4444 {
4445 /*
4446 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4447 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4448 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4449 * loaded in protected-mode have their attribute as 0.
4450 */
4451 if (!u32Access)
4452 u32Access = X86DESCATTR_UNUSABLE;
4453 }
4454
4455 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4456 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4457 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4458
4459 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4460 AssertRCReturn(rc, rc);
4461 return rc;
4462}
4463
4464
4465/**
4466 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4467 * into the guest-state area in the VMCS.
4468 *
4469 * @returns VBox status code.
4470 * @param pVM Pointer to the VM.
4471 * @param pVCPU Pointer to the VMCPU.
4472 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4473 * out-of-sync. Make sure to update the required fields
4474 * before using them.
4475 *
4476 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4477 * @remarks No-long-jump zone!!!
4478 */
4479static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4480{
4481 int rc = VERR_INTERNAL_ERROR_5;
4482 PVM pVM = pVCpu->CTX_SUFF(pVM);
4483
4484 /*
4485 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4486 */
4487 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4488 {
4489 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4490 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4491 {
4492 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4493 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4494 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4495 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4496 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4498 }
4499
4500#ifdef VBOX_WITH_REM
4501 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4502 {
4503 Assert(pVM->hm.s.vmx.pRealModeTSS);
4504 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4505 if ( pVCpu->hm.s.vmx.fWasInRealMode
4506 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4507 {
4508 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4509 in real-mode (e.g. OpenBSD 4.0) */
4510 REMFlushTBs(pVM);
4511 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4512 pVCpu->hm.s.vmx.fWasInRealMode = false;
4513 }
4514 }
4515#endif
4516 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4517 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4518 AssertRCReturn(rc, rc);
4519 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4520 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4521 AssertRCReturn(rc, rc);
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4523 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4526 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4529 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4532 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4533 AssertRCReturn(rc, rc);
4534
4535#ifdef VBOX_STRICT
4536 /* Validate. */
4537 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4538#endif
4539
4540 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4541 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4542 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4543 }
4544
4545 /*
4546 * Guest TR.
4547 */
4548 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4549 {
4550 /*
4551 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4552 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4553 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4554 */
4555 uint16_t u16Sel = 0;
4556 uint32_t u32Limit = 0;
4557 uint64_t u64Base = 0;
4558 uint32_t u32AccessRights = 0;
4559
4560 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4561 {
4562 u16Sel = pMixedCtx->tr.Sel;
4563 u32Limit = pMixedCtx->tr.u32Limit;
4564 u64Base = pMixedCtx->tr.u64Base;
4565 u32AccessRights = pMixedCtx->tr.Attr.u;
4566 }
4567 else
4568 {
4569 Assert(pVM->hm.s.vmx.pRealModeTSS);
4570 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4571
4572 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4573 RTGCPHYS GCPhys;
4574 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4575 AssertRCReturn(rc, rc);
4576
4577 X86DESCATTR DescAttr;
4578 DescAttr.u = 0;
4579 DescAttr.n.u1Present = 1;
4580 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4581
4582 u16Sel = 0;
4583 u32Limit = HM_VTX_TSS_SIZE;
4584 u64Base = GCPhys; /* in real-mode phys = virt. */
4585 u32AccessRights = DescAttr.u;
4586 }
4587
4588 /* Validate. */
4589 Assert(!(u16Sel & RT_BIT(2)));
4590 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4591 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4592 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4593 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4594 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4595 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4596 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4597 Assert( (u32Limit & 0xfff) == 0xfff
4598 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4599 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4600 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4601
4602 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4603 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4604 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4605 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4606
4607 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4608 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4609 }
4610
4611 /*
4612 * Guest GDTR.
4613 */
4614 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4615 {
4616 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4617 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4618
4619 /* Validate. */
4620 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4621
4622 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4623 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4624 }
4625
4626 /*
4627 * Guest LDTR.
4628 */
4629 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4630 {
4631 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4632 uint32_t u32Access = 0;
4633 if (!pMixedCtx->ldtr.Attr.u)
4634 u32Access = X86DESCATTR_UNUSABLE;
4635 else
4636 u32Access = pMixedCtx->ldtr.Attr.u;
4637
4638 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4639 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4640 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4641 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4642
4643 /* Validate. */
4644 if (!(u32Access & X86DESCATTR_UNUSABLE))
4645 {
4646 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4647 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4648 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4649 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4650 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4651 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4652 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4653 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4654 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4655 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4656 }
4657
4658 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4659 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4660 }
4661
4662 /*
4663 * Guest IDTR.
4664 */
4665 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4666 {
4667 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4668 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4669
4670 /* Validate. */
4671 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4672
4673 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4674 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4675 }
4676
4677 return VINF_SUCCESS;
4678}
4679
4680
4681/**
4682 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4683 * areas. These MSRs will automatically be loaded to the host CPU on every
4684 * successful VM entry and stored from the host CPU on every successful VM-exit.
4685 *
4686 * This also creates/updates MSR slots for the host MSRs. The actual host
4687 * MSR values are -not- updated here for performance reasons. See
4688 * hmR0VmxSaveHostMsrs().
4689 *
4690 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4691 *
4692 * @returns VBox status code.
4693 * @param pVCpu Pointer to the VMCPU.
4694 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4695 * out-of-sync. Make sure to update the required fields
4696 * before using them.
4697 *
4698 * @remarks No-long-jump zone!!!
4699 */
4700static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4701{
4702 AssertPtr(pVCpu);
4703 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4704
4705 /*
4706 * MSRs that we use the auto-load/store MSR area in the VMCS.
4707 */
4708 PVM pVM = pVCpu->CTX_SUFF(pVM);
4709 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4710 {
4711 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4712#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4713 if (pVM->hm.s.fAllow64BitGuests)
4714 {
4715 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4716 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4717 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4718 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4719# ifdef DEBUG
4720 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4721 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4722 {
4723 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4724 pMsr->u64Value));
4725 }
4726# endif
4727 }
4728#endif
4729 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4730 }
4731
4732 /*
4733 * Guest Sysenter MSRs.
4734 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4735 * VM-exits on WRMSRs for these MSRs.
4736 */
4737 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4738 {
4739 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4740 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4741 }
4742
4743 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4744 {
4745 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4746 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4747 }
4748
4749 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4750 {
4751 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4752 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4753 }
4754
4755 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4756 {
4757 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4758 {
4759 /*
4760 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4761 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4762 */
4763 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4764 {
4765 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4766 AssertRCReturn(rc,rc);
4767 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4768 }
4769 else
4770 {
4771 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4772 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4773 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4774 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4775 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4776 }
4777 }
4778 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4779 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4780 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4781 }
4782
4783 return VINF_SUCCESS;
4784}
4785
4786
4787/**
4788 * Loads the guest activity state into the guest-state area in the VMCS.
4789 *
4790 * @returns VBox status code.
4791 * @param pVCpu Pointer to the VMCPU.
4792 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4793 * out-of-sync. Make sure to update the required fields
4794 * before using them.
4795 *
4796 * @remarks No-long-jump zone!!!
4797 */
4798static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4799{
4800 NOREF(pCtx);
4801 /** @todo See if we can make use of other states, e.g.
4802 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4803 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4804 {
4805 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4806 AssertRCReturn(rc, rc);
4807
4808 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4809 }
4810 return VINF_SUCCESS;
4811}
4812
4813
4814/**
4815 * Sets up the appropriate function to run guest code.
4816 *
4817 * @returns VBox status code.
4818 * @param pVCpu Pointer to the VMCPU.
4819 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4820 * out-of-sync. Make sure to update the required fields
4821 * before using them.
4822 *
4823 * @remarks No-long-jump zone!!!
4824 */
4825static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4826{
4827 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4828 {
4829#ifndef VBOX_ENABLE_64_BITS_GUESTS
4830 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4831#endif
4832 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4833#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4834 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4835 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4836 {
4837 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4838 {
4839 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4840 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4841 | HM_CHANGED_VMX_EXIT_CTLS
4842 | HM_CHANGED_VMX_ENTRY_CTLS
4843 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4844 }
4845 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4846 }
4847#else
4848 /* 64-bit host or hybrid host. */
4849 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4850#endif
4851 }
4852 else
4853 {
4854 /* Guest is not in long mode, use the 32-bit handler. */
4855#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4856 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4857 {
4858 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4859 {
4860 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4861 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4862 | HM_CHANGED_VMX_EXIT_CTLS
4863 | HM_CHANGED_VMX_ENTRY_CTLS
4864 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4865 }
4866 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4867 }
4868#else
4869 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4870#endif
4871 }
4872 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4873 return VINF_SUCCESS;
4874}
4875
4876
4877/**
4878 * Wrapper for running the guest code in VT-x.
4879 *
4880 * @returns VBox strict status code.
4881 * @param pVM Pointer to the VM.
4882 * @param pVCpu Pointer to the VMCPU.
4883 * @param pCtx Pointer to the guest-CPU context.
4884 *
4885 * @remarks No-long-jump zone!!!
4886 */
4887DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4888{
4889 /*
4890 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4891 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4892 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4893 */
4894 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4895 /** @todo Add stats for resume vs launch. */
4896#ifdef VBOX_WITH_KERNEL_USING_XMM
4897 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4898#else
4899 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4900#endif
4901}
4902
4903
4904/**
4905 * Reports world-switch error and dumps some useful debug info.
4906 *
4907 * @param pVM Pointer to the VM.
4908 * @param pVCpu Pointer to the VMCPU.
4909 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4910 * @param pCtx Pointer to the guest-CPU context.
4911 * @param pVmxTransient Pointer to the VMX transient structure (only
4912 * exitReason updated).
4913 */
4914static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4915{
4916 Assert(pVM);
4917 Assert(pVCpu);
4918 Assert(pCtx);
4919 Assert(pVmxTransient);
4920 HMVMX_ASSERT_PREEMPT_SAFE();
4921
4922 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4923 switch (rcVMRun)
4924 {
4925 case VERR_VMX_INVALID_VMXON_PTR:
4926 AssertFailed();
4927 break;
4928 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4929 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4930 {
4931 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4932 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4933 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4934 AssertRC(rc);
4935
4936 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4937 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4938 Cannot do it here as we may have been long preempted. */
4939
4940#ifdef VBOX_STRICT
4941 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4942 pVmxTransient->uExitReason));
4943 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4944 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4945 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4946 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4947 else
4948 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4949 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4950 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4951
4952 /* VMX control bits. */
4953 uint32_t u32Val;
4954 uint64_t u64Val;
4955 HMVMXHCUINTREG uHCReg;
4956 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4957 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4958 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4959 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4960 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4961 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4962 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4963 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4964 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4965 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4988 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4989 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4990 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4991 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4992 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4993 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4994 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4995 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4996 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4997 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4998
4999 /* Guest bits. */
5000 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5001 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5002 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5003 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5004 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5005 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5006 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5007 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5008
5009 /* Host bits. */
5010 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5011 Log4(("Host CR0 %#RHr\n", uHCReg));
5012 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5013 Log4(("Host CR3 %#RHr\n", uHCReg));
5014 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5015 Log4(("Host CR4 %#RHr\n", uHCReg));
5016
5017 RTGDTR HostGdtr;
5018 PCX86DESCHC pDesc;
5019 ASMGetGDTR(&HostGdtr);
5020 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5021 Log4(("Host CS %#08x\n", u32Val));
5022 if (u32Val < HostGdtr.cbGdt)
5023 {
5024 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5025 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5026 }
5027
5028 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5029 Log4(("Host DS %#08x\n", u32Val));
5030 if (u32Val < HostGdtr.cbGdt)
5031 {
5032 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5033 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5034 }
5035
5036 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5037 Log4(("Host ES %#08x\n", u32Val));
5038 if (u32Val < HostGdtr.cbGdt)
5039 {
5040 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5041 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5042 }
5043
5044 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5045 Log4(("Host FS %#08x\n", u32Val));
5046 if (u32Val < HostGdtr.cbGdt)
5047 {
5048 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5049 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5050 }
5051
5052 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5053 Log4(("Host GS %#08x\n", u32Val));
5054 if (u32Val < HostGdtr.cbGdt)
5055 {
5056 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5057 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5058 }
5059
5060 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5061 Log4(("Host SS %#08x\n", u32Val));
5062 if (u32Val < HostGdtr.cbGdt)
5063 {
5064 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5065 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5066 }
5067
5068 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5069 Log4(("Host TR %#08x\n", u32Val));
5070 if (u32Val < HostGdtr.cbGdt)
5071 {
5072 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5073 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5074 }
5075
5076 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5077 Log4(("Host TR Base %#RHv\n", uHCReg));
5078 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5079 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5080 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5081 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5082 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5083 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5085 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5087 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5089 Log4(("Host RSP %#RHv\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5091 Log4(("Host RIP %#RHv\n", uHCReg));
5092# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5093 if (HMVMX_IS_64BIT_HOST_MODE())
5094 {
5095 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5096 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5097 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5098 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5099 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5100 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5101 }
5102# endif
5103#endif /* VBOX_STRICT */
5104 break;
5105 }
5106
5107 default:
5108 /* Impossible */
5109 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5110 break;
5111 }
5112 NOREF(pVM); NOREF(pCtx);
5113}
5114
5115
5116#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5117#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5118# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5119#endif
5120#ifdef VBOX_STRICT
5121static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5122{
5123 switch (idxField)
5124 {
5125 case VMX_VMCS_GUEST_RIP:
5126 case VMX_VMCS_GUEST_RSP:
5127 case VMX_VMCS_GUEST_SYSENTER_EIP:
5128 case VMX_VMCS_GUEST_SYSENTER_ESP:
5129 case VMX_VMCS_GUEST_GDTR_BASE:
5130 case VMX_VMCS_GUEST_IDTR_BASE:
5131 case VMX_VMCS_GUEST_CS_BASE:
5132 case VMX_VMCS_GUEST_DS_BASE:
5133 case VMX_VMCS_GUEST_ES_BASE:
5134 case VMX_VMCS_GUEST_FS_BASE:
5135 case VMX_VMCS_GUEST_GS_BASE:
5136 case VMX_VMCS_GUEST_SS_BASE:
5137 case VMX_VMCS_GUEST_LDTR_BASE:
5138 case VMX_VMCS_GUEST_TR_BASE:
5139 case VMX_VMCS_GUEST_CR3:
5140 return true;
5141 }
5142 return false;
5143}
5144
5145static bool hmR0VmxIsValidReadField(uint32_t idxField)
5146{
5147 switch (idxField)
5148 {
5149 /* Read-only fields. */
5150 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5151 return true;
5152 }
5153 /* Remaining readable fields should also be writable. */
5154 return hmR0VmxIsValidWriteField(idxField);
5155}
5156#endif /* VBOX_STRICT */
5157
5158
5159/**
5160 * Executes the specified handler in 64-bit mode.
5161 *
5162 * @returns VBox status code.
5163 * @param pVM Pointer to the VM.
5164 * @param pVCpu Pointer to the VMCPU.
5165 * @param pCtx Pointer to the guest CPU context.
5166 * @param enmOp The operation to perform.
5167 * @param cbParam Number of parameters.
5168 * @param paParam Array of 32-bit parameters.
5169 */
5170VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5171 uint32_t *paParam)
5172{
5173 int rc, rc2;
5174 PHMGLOBALCPUINFO pCpu;
5175 RTHCPHYS HCPhysCpuPage;
5176 RTCCUINTREG uOldEflags;
5177
5178 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5179 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5180 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5181 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5182
5183#ifdef VBOX_STRICT
5184 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5185 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5186
5187 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5188 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5189#endif
5190
5191 /* Disable interrupts. */
5192 uOldEflags = ASMIntDisableFlags();
5193
5194#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5195 RTCPUID idHostCpu = RTMpCpuId();
5196 CPUMR0SetLApic(pVCpu, idHostCpu);
5197#endif
5198
5199 pCpu = HMR0GetCurrentCpu();
5200 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5201
5202 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5203 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5204
5205 /* Leave VMX Root Mode. */
5206 VMXDisable();
5207
5208 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5209
5210 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5211 CPUMSetHyperEIP(pVCpu, enmOp);
5212 for (int i = (int)cbParam - 1; i >= 0; i--)
5213 CPUMPushHyper(pVCpu, paParam[i]);
5214
5215 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5216
5217 /* Call the switcher. */
5218 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5219 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5220
5221 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5222 /* Make sure the VMX instructions don't cause #UD faults. */
5223 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5224
5225 /* Re-enter VMX Root Mode */
5226 rc2 = VMXEnable(HCPhysCpuPage);
5227 if (RT_FAILURE(rc2))
5228 {
5229 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5230 ASMSetFlags(uOldEflags);
5231 return rc2;
5232 }
5233
5234 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5235 AssertRC(rc2);
5236 Assert(!(ASMGetFlags() & X86_EFL_IF));
5237 ASMSetFlags(uOldEflags);
5238 return rc;
5239}
5240
5241
5242/**
5243 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5244 * supporting 64-bit guests.
5245 *
5246 * @returns VBox status code.
5247 * @param fResume Whether to VMLAUNCH or VMRESUME.
5248 * @param pCtx Pointer to the guest-CPU context.
5249 * @param pCache Pointer to the VMCS cache.
5250 * @param pVM Pointer to the VM.
5251 * @param pVCpu Pointer to the VMCPU.
5252 */
5253DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5254{
5255 uint32_t aParam[6];
5256 PHMGLOBALCPUINFO pCpu = NULL;
5257 RTHCPHYS HCPhysCpuPage = 0;
5258 int rc = VERR_INTERNAL_ERROR_5;
5259
5260 pCpu = HMR0GetCurrentCpu();
5261 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5262
5263#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5264 pCache->uPos = 1;
5265 pCache->interPD = PGMGetInterPaeCR3(pVM);
5266 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5267#endif
5268
5269#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5270 pCache->TestIn.HCPhysCpuPage = 0;
5271 pCache->TestIn.HCPhysVmcs = 0;
5272 pCache->TestIn.pCache = 0;
5273 pCache->TestOut.HCPhysVmcs = 0;
5274 pCache->TestOut.pCache = 0;
5275 pCache->TestOut.pCtx = 0;
5276 pCache->TestOut.eflags = 0;
5277#endif
5278
5279 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5280 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5281 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5282 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5283 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5284 aParam[5] = 0;
5285
5286#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5287 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5288 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5289#endif
5290 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5291
5292#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5293 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5294 Assert(pCtx->dr[4] == 10);
5295 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5296#endif
5297
5298#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5299 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5300 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5301 pVCpu->hm.s.vmx.HCPhysVmcs));
5302 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5303 pCache->TestOut.HCPhysVmcs));
5304 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5305 pCache->TestOut.pCache));
5306 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5307 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5308 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5309 pCache->TestOut.pCtx));
5310 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5311#endif
5312 return rc;
5313}
5314
5315
5316/**
5317 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5318 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5319 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5320 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5321 *
5322 * @returns VBox status code.
5323 * @param pVM Pointer to the VM.
5324 * @param pVCpu Pointer to the VMCPU.
5325 */
5326static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5327{
5328#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5329{ \
5330 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5331 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5332 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5333 ++cReadFields; \
5334}
5335
5336 AssertPtr(pVM);
5337 AssertPtr(pVCpu);
5338 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5339 uint32_t cReadFields = 0;
5340
5341 /*
5342 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5343 * and serve to indicate exceptions to the rules.
5344 */
5345
5346 /* Guest-natural selector base fields. */
5347#if 0
5348 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5349 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5350 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5351#endif
5352 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5354 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5364#if 0
5365 /* Unused natural width guest-state fields. */
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5368#endif
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5371
5372 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5373#if 0
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5383#endif
5384
5385 /* Natural width guest-state fields. */
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5387#if 0
5388 /* Currently unused field. */
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5390#endif
5391
5392 if (pVM->hm.s.fNestedPaging)
5393 {
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5395 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5396 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5397 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5398 }
5399 else
5400 {
5401 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5402 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5403 }
5404
5405#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5406 return VINF_SUCCESS;
5407}
5408
5409
5410/**
5411 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5412 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5413 * darwin, running 64-bit guests).
5414 *
5415 * @returns VBox status code.
5416 * @param pVCpu Pointer to the VMCPU.
5417 * @param idxField The VMCS field encoding.
5418 * @param u64Val 16, 32 or 64-bit value.
5419 */
5420VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5421{
5422 int rc;
5423 switch (idxField)
5424 {
5425 /*
5426 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5427 */
5428 /* 64-bit Control fields. */
5429 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5430 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5431 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5432 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5433 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5434 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5435 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5436 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5437 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5438 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5439 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5440 case VMX_VMCS64_CTRL_EPTP_FULL:
5441 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5442 /* 64-bit Guest-state fields. */
5443 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5444 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5445 case VMX_VMCS64_GUEST_PAT_FULL:
5446 case VMX_VMCS64_GUEST_EFER_FULL:
5447 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5448 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5449 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5450 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5451 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5452 /* 64-bit Host-state fields. */
5453 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5454 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5455 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5456 {
5457 rc = VMXWriteVmcs32(idxField, u64Val);
5458 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5459 break;
5460 }
5461
5462 /*
5463 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5464 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5465 */
5466 /* Natural-width Guest-state fields. */
5467 case VMX_VMCS_GUEST_CR3:
5468 case VMX_VMCS_GUEST_ES_BASE:
5469 case VMX_VMCS_GUEST_CS_BASE:
5470 case VMX_VMCS_GUEST_SS_BASE:
5471 case VMX_VMCS_GUEST_DS_BASE:
5472 case VMX_VMCS_GUEST_FS_BASE:
5473 case VMX_VMCS_GUEST_GS_BASE:
5474 case VMX_VMCS_GUEST_LDTR_BASE:
5475 case VMX_VMCS_GUEST_TR_BASE:
5476 case VMX_VMCS_GUEST_GDTR_BASE:
5477 case VMX_VMCS_GUEST_IDTR_BASE:
5478 case VMX_VMCS_GUEST_RSP:
5479 case VMX_VMCS_GUEST_RIP:
5480 case VMX_VMCS_GUEST_SYSENTER_ESP:
5481 case VMX_VMCS_GUEST_SYSENTER_EIP:
5482 {
5483 if (!(u64Val >> 32))
5484 {
5485 /* If this field is 64-bit, VT-x will zero out the top bits. */
5486 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5487 }
5488 else
5489 {
5490 /* Assert that only the 32->64 switcher case should ever come here. */
5491 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5492 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5493 }
5494 break;
5495 }
5496
5497 default:
5498 {
5499 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5500 rc = VERR_INVALID_PARAMETER;
5501 break;
5502 }
5503 }
5504 AssertRCReturn(rc, rc);
5505 return rc;
5506}
5507
5508
5509/**
5510 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5511 * hosts (except darwin) for 64-bit guests.
5512 *
5513 * @param pVCpu Pointer to the VMCPU.
5514 * @param idxField The VMCS field encoding.
5515 * @param u64Val 16, 32 or 64-bit value.
5516 */
5517VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5518{
5519 AssertPtr(pVCpu);
5520 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5521
5522 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5523 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5524
5525 /* Make sure there are no duplicates. */
5526 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5527 {
5528 if (pCache->Write.aField[i] == idxField)
5529 {
5530 pCache->Write.aFieldVal[i] = u64Val;
5531 return VINF_SUCCESS;
5532 }
5533 }
5534
5535 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5536 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5537 pCache->Write.cValidEntries++;
5538 return VINF_SUCCESS;
5539}
5540
5541/* Enable later when the assembly code uses these as callbacks. */
5542#if 0
5543/*
5544 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5545 *
5546 * @param pVCpu Pointer to the VMCPU.
5547 * @param pCache Pointer to the VMCS cache.
5548 *
5549 * @remarks No-long-jump zone!!!
5550 */
5551VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5552{
5553 AssertPtr(pCache);
5554 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5555 {
5556 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5557 AssertRC(rc);
5558 }
5559 pCache->Write.cValidEntries = 0;
5560}
5561
5562
5563/**
5564 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5565 *
5566 * @param pVCpu Pointer to the VMCPU.
5567 * @param pCache Pointer to the VMCS cache.
5568 *
5569 * @remarks No-long-jump zone!!!
5570 */
5571VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5572{
5573 AssertPtr(pCache);
5574 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5575 {
5576 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5577 AssertRC(rc);
5578 }
5579}
5580#endif
5581#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5582
5583
5584/**
5585 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5586 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5587 * timer.
5588 *
5589 * @returns VBox status code.
5590 * @param pVCpu Pointer to the VMCPU.
5591 *
5592 * @remarks No-long-jump zone!!!
5593 */
5594static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5595{
5596 int rc = VERR_INTERNAL_ERROR_5;
5597 bool fOffsettedTsc = false;
5598 bool fParavirtTsc = false;
5599 PVM pVM = pVCpu->CTX_SUFF(pVM);
5600 if (pVM->hm.s.vmx.fUsePreemptTimer)
5601 {
5602 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5603 &pVCpu->hm.s.vmx.u64TSCOffset);
5604
5605 /* Make sure the returned values have sane upper and lower boundaries. */
5606 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5607 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5608 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5609 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5610
5611 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5612 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5613 }
5614 else
5615 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5616
5617#if 1
5618 if (fParavirtTsc)
5619 {
5620#if 1
5621 uint64_t const u64CurTsc = ASMReadTSC();
5622 uint64_t const u64LastTick = TMCpuTickGetLastSeen(pVCpu);
5623 if (u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset < u64LastTick)
5624 {
5625 pVCpu->hm.s.vmx.u64TSCOffset = (u64LastTick - u64CurTsc);
5626 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffsetAdjusted);
5627 }
5628
5629 Assert(u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset >= u64LastTick);
5630#endif
5631 rc = GIMR0UpdateParavirtTsc(pVM, pVCpu->hm.s.vmx.u64TSCOffset);
5632 AssertRC(rc);
5633 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5634 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRC(rc);
5635
5636 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5637 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5638 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5639 }
5640 else
5641#else
5642 if (fParavirtTsc)
5643 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5644#endif
5645 if (fOffsettedTsc)
5646 {
5647 uint64_t u64CurTSC = ASMReadTSC();
5648 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5649 {
5650 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5651 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5652
5653 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5654 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5655 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5656 }
5657 else
5658 {
5659 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5660 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5661 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5662 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5663 }
5664 }
5665 else
5666 {
5667 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5668 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5669 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5670 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5671 }
5672}
5673
5674
5675/**
5676 * Determines if an exception is a contributory exception. Contributory
5677 * exceptions are ones which can cause double-faults. Page-fault is
5678 * intentionally not included here as it's a conditional contributory exception.
5679 *
5680 * @returns true if the exception is contributory, false otherwise.
5681 * @param uVector The exception vector.
5682 */
5683DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5684{
5685 switch (uVector)
5686 {
5687 case X86_XCPT_GP:
5688 case X86_XCPT_SS:
5689 case X86_XCPT_NP:
5690 case X86_XCPT_TS:
5691 case X86_XCPT_DE:
5692 return true;
5693 default:
5694 break;
5695 }
5696 return false;
5697}
5698
5699
5700/**
5701 * Sets an event as a pending event to be injected into the guest.
5702 *
5703 * @param pVCpu Pointer to the VMCPU.
5704 * @param u32IntInfo The VM-entry interruption-information field.
5705 * @param cbInstr The VM-entry instruction length in bytes (for software
5706 * interrupts, exceptions and privileged software
5707 * exceptions).
5708 * @param u32ErrCode The VM-entry exception error code.
5709 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5710 * page-fault.
5711 *
5712 * @remarks Statistics counter assumes this is a guest event being injected or
5713 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5714 * always incremented.
5715 */
5716DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5717 RTGCUINTPTR GCPtrFaultAddress)
5718{
5719 Assert(!pVCpu->hm.s.Event.fPending);
5720 pVCpu->hm.s.Event.fPending = true;
5721 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5722 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5723 pVCpu->hm.s.Event.cbInstr = cbInstr;
5724 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5725
5726 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5727}
5728
5729
5730/**
5731 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5732 *
5733 * @param pVCpu Pointer to the VMCPU.
5734 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5735 * out-of-sync. Make sure to update the required fields
5736 * before using them.
5737 */
5738DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5739{
5740 NOREF(pMixedCtx);
5741 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5742 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5743 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5744 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5745}
5746
5747
5748/**
5749 * Handle a condition that occurred while delivering an event through the guest
5750 * IDT.
5751 *
5752 * @returns VBox status code (informational error codes included).
5753 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5754 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5755 * continue execution of the guest which will delivery the #DF.
5756 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5757 *
5758 * @param pVCpu Pointer to the VMCPU.
5759 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5760 * out-of-sync. Make sure to update the required fields
5761 * before using them.
5762 * @param pVmxTransient Pointer to the VMX transient structure.
5763 *
5764 * @remarks No-long-jump zone!!!
5765 */
5766static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5767{
5768 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5769
5770 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5771 AssertRCReturn(rc, rc);
5772 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5773 {
5774 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5775 AssertRCReturn(rc, rc);
5776
5777 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5778 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5779
5780 typedef enum
5781 {
5782 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5783 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5784 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5785 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5786 } VMXREFLECTXCPT;
5787
5788 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5789 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5790 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5791 {
5792 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5793 {
5794 enmReflect = VMXREFLECTXCPT_XCPT;
5795#ifdef VBOX_STRICT
5796 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5797 && uExitVector == X86_XCPT_PF)
5798 {
5799 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5800 }
5801#endif
5802 if ( uExitVector == X86_XCPT_PF
5803 && uIdtVector == X86_XCPT_PF)
5804 {
5805 pVmxTransient->fVectoringPF = true;
5806 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5807 }
5808 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5809 && hmR0VmxIsContributoryXcpt(uExitVector)
5810 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5811 || uIdtVector == X86_XCPT_PF))
5812 {
5813 enmReflect = VMXREFLECTXCPT_DF;
5814 }
5815 else if (uIdtVector == X86_XCPT_DF)
5816 enmReflect = VMXREFLECTXCPT_TF;
5817 }
5818 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5819 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5820 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5821 {
5822 /*
5823 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5824 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5825 */
5826 enmReflect = VMXREFLECTXCPT_XCPT;
5827 }
5828 }
5829 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5830 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5831 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5832 {
5833 /*
5834 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5835 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5836 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5837 */
5838 enmReflect = VMXREFLECTXCPT_XCPT;
5839 }
5840
5841 /*
5842 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5843 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5844 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5845 *
5846 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5847 */
5848 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5849 && enmReflect == VMXREFLECTXCPT_XCPT
5850 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5851 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5852 {
5853 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5854 }
5855
5856 switch (enmReflect)
5857 {
5858 case VMXREFLECTXCPT_XCPT:
5859 {
5860 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5861 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5862 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5863
5864 uint32_t u32ErrCode = 0;
5865 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5866 {
5867 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5868 AssertRCReturn(rc, rc);
5869 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5870 }
5871
5872 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5873 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5874 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5875 rc = VINF_SUCCESS;
5876 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5877 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5878
5879 break;
5880 }
5881
5882 case VMXREFLECTXCPT_DF:
5883 {
5884 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5885 rc = VINF_HM_DOUBLE_FAULT;
5886 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5887 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5888
5889 break;
5890 }
5891
5892 case VMXREFLECTXCPT_TF:
5893 {
5894 rc = VINF_EM_RESET;
5895 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5896 uExitVector));
5897 break;
5898 }
5899
5900 default:
5901 Assert(rc == VINF_SUCCESS);
5902 break;
5903 }
5904 }
5905 else if ( VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5906 && uExitVector != X86_XCPT_DF
5907 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5908 {
5909 /*
5910 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5911 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5912 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5913 */
5914 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5915 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5916 }
5917
5918 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5919 return rc;
5920}
5921
5922
5923/**
5924 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5925 *
5926 * @returns VBox status code.
5927 * @param pVCpu Pointer to the VMCPU.
5928 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5929 * out-of-sync. Make sure to update the required fields
5930 * before using them.
5931 *
5932 * @remarks No-long-jump zone!!!
5933 */
5934static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5935{
5936 NOREF(pMixedCtx);
5937
5938 /*
5939 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5940 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5941 */
5942 VMMRZCallRing3Disable(pVCpu);
5943 HM_DISABLE_PREEMPT_IF_NEEDED();
5944
5945 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5946 {
5947 uint32_t uVal = 0;
5948 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5949 AssertRCReturn(rc, rc);
5950
5951 uint32_t uShadow = 0;
5952 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5953 AssertRCReturn(rc, rc);
5954
5955 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5956 CPUMSetGuestCR0(pVCpu, uVal);
5957 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5958 }
5959
5960 HM_RESTORE_PREEMPT_IF_NEEDED();
5961 VMMRZCallRing3Enable(pVCpu);
5962 return VINF_SUCCESS;
5963}
5964
5965
5966/**
5967 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5968 *
5969 * @returns VBox status code.
5970 * @param pVCpu Pointer to the VMCPU.
5971 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5972 * out-of-sync. Make sure to update the required fields
5973 * before using them.
5974 *
5975 * @remarks No-long-jump zone!!!
5976 */
5977static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5978{
5979 NOREF(pMixedCtx);
5980
5981 int rc = VINF_SUCCESS;
5982 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5983 {
5984 uint32_t uVal = 0;
5985 uint32_t uShadow = 0;
5986 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5987 AssertRCReturn(rc, rc);
5988 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5989 AssertRCReturn(rc, rc);
5990
5991 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5992 CPUMSetGuestCR4(pVCpu, uVal);
5993 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5994 }
5995 return rc;
5996}
5997
5998
5999/**
6000 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6001 *
6002 * @returns VBox status code.
6003 * @param pVCpu Pointer to the VMCPU.
6004 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6005 * out-of-sync. Make sure to update the required fields
6006 * before using them.
6007 *
6008 * @remarks No-long-jump zone!!!
6009 */
6010static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6011{
6012 int rc = VINF_SUCCESS;
6013 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6014 {
6015 uint64_t u64Val = 0;
6016 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6017 AssertRCReturn(rc, rc);
6018
6019 pMixedCtx->rip = u64Val;
6020 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6021 }
6022 return rc;
6023}
6024
6025
6026/**
6027 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6028 *
6029 * @returns VBox status code.
6030 * @param pVCpu Pointer to the VMCPU.
6031 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6032 * out-of-sync. Make sure to update the required fields
6033 * before using them.
6034 *
6035 * @remarks No-long-jump zone!!!
6036 */
6037static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6038{
6039 int rc = VINF_SUCCESS;
6040 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6041 {
6042 uint64_t u64Val = 0;
6043 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6044 AssertRCReturn(rc, rc);
6045
6046 pMixedCtx->rsp = u64Val;
6047 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6048 }
6049 return rc;
6050}
6051
6052
6053/**
6054 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6055 *
6056 * @returns VBox status code.
6057 * @param pVCpu Pointer to the VMCPU.
6058 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6059 * out-of-sync. Make sure to update the required fields
6060 * before using them.
6061 *
6062 * @remarks No-long-jump zone!!!
6063 */
6064static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6065{
6066 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6067 {
6068 uint32_t uVal = 0;
6069 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6070 AssertRCReturn(rc, rc);
6071
6072 pMixedCtx->eflags.u32 = uVal;
6073 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6074 {
6075 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6076 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6077
6078 pMixedCtx->eflags.Bits.u1VM = 0;
6079 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6080 }
6081
6082 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6083 }
6084 return VINF_SUCCESS;
6085}
6086
6087
6088/**
6089 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6090 * guest-CPU context.
6091 */
6092DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6093{
6094 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6095 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6096 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6097 return rc;
6098}
6099
6100
6101/**
6102 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6103 * from the guest-state area in the VMCS.
6104 *
6105 * @param pVCpu Pointer to the VMCPU.
6106 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6107 * out-of-sync. Make sure to update the required fields
6108 * before using them.
6109 *
6110 * @remarks No-long-jump zone!!!
6111 */
6112static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6113{
6114 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6115 {
6116 uint32_t uIntrState = 0;
6117 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6118 AssertRC(rc);
6119
6120 if (!uIntrState)
6121 {
6122 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6123 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6124
6125 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6126 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6127 }
6128 else
6129 {
6130 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6131 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6132 {
6133 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6134 AssertRC(rc);
6135 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6136 AssertRC(rc);
6137
6138 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6139 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6140 }
6141 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6142 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6143
6144 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6145 {
6146 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6147 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6148 }
6149 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6150 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6151 }
6152
6153 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6154 }
6155}
6156
6157
6158/**
6159 * Saves the guest's activity state.
6160 *
6161 * @returns VBox status code.
6162 * @param pVCpu Pointer to the VMCPU.
6163 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6164 * out-of-sync. Make sure to update the required fields
6165 * before using them.
6166 *
6167 * @remarks No-long-jump zone!!!
6168 */
6169static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6170{
6171 NOREF(pMixedCtx);
6172 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6173 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6174 return VINF_SUCCESS;
6175}
6176
6177
6178/**
6179 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6180 * the current VMCS into the guest-CPU context.
6181 *
6182 * @returns VBox status code.
6183 * @param pVCpu Pointer to the VMCPU.
6184 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6185 * out-of-sync. Make sure to update the required fields
6186 * before using them.
6187 *
6188 * @remarks No-long-jump zone!!!
6189 */
6190static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6191{
6192 int rc = VINF_SUCCESS;
6193 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6194 {
6195 uint32_t u32Val = 0;
6196 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6197 pMixedCtx->SysEnter.cs = u32Val;
6198 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6199 }
6200
6201 uint64_t u64Val = 0;
6202 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6203 {
6204 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6205 pMixedCtx->SysEnter.eip = u64Val;
6206 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6207 }
6208 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6209 {
6210 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6211 pMixedCtx->SysEnter.esp = u64Val;
6212 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6213 }
6214 return rc;
6215}
6216
6217
6218/**
6219 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6220 * the CPU back into the guest-CPU context.
6221 *
6222 * @returns VBox status code.
6223 * @param pVCpu Pointer to the VMCPU.
6224 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6225 * out-of-sync. Make sure to update the required fields
6226 * before using them.
6227 *
6228 * @remarks No-long-jump zone!!!
6229 */
6230static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6231{
6232#if HC_ARCH_BITS == 64
6233 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6234 {
6235 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6236 VMMRZCallRing3Disable(pVCpu);
6237 HM_DISABLE_PREEMPT_IF_NEEDED();
6238
6239 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6240 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6241 {
6242 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6243 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6244 }
6245
6246 HM_RESTORE_PREEMPT_IF_NEEDED();
6247 VMMRZCallRing3Enable(pVCpu);
6248 }
6249 else
6250 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6251#else
6252 NOREF(pMixedCtx);
6253 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6254#endif
6255
6256 return VINF_SUCCESS;
6257}
6258
6259
6260/**
6261 * Saves the auto load/store'd guest MSRs from the current VMCS into
6262 * the guest-CPU context.
6263 *
6264 * @returns VBox status code.
6265 * @param pVCpu Pointer to the VMCPU.
6266 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6267 * out-of-sync. Make sure to update the required fields
6268 * before using them.
6269 *
6270 * @remarks No-long-jump zone!!!
6271 */
6272static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6273{
6274 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6275 return VINF_SUCCESS;
6276
6277 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6278 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6279 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6280 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6281 {
6282 switch (pMsr->u32Msr)
6283 {
6284 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6285 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6286 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6287 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6288 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6289 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6290 break;
6291
6292 default:
6293 {
6294 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6295 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6296 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6297 }
6298 }
6299 }
6300
6301 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6302 return VINF_SUCCESS;
6303}
6304
6305
6306/**
6307 * Saves the guest control registers from the current VMCS into the guest-CPU
6308 * context.
6309 *
6310 * @returns VBox status code.
6311 * @param pVCpu Pointer to the VMCPU.
6312 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6313 * out-of-sync. Make sure to update the required fields
6314 * before using them.
6315 *
6316 * @remarks No-long-jump zone!!!
6317 */
6318static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6319{
6320 /* Guest CR0. Guest FPU. */
6321 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6322 AssertRCReturn(rc, rc);
6323
6324 /* Guest CR4. */
6325 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6326 AssertRCReturn(rc, rc);
6327
6328 /* Guest CR2 - updated always during the world-switch or in #PF. */
6329 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6330 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6331 {
6332 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6333 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6334
6335 PVM pVM = pVCpu->CTX_SUFF(pVM);
6336 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6337 || ( pVM->hm.s.fNestedPaging
6338 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6339 {
6340 uint64_t u64Val = 0;
6341 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6342 if (pMixedCtx->cr3 != u64Val)
6343 {
6344 CPUMSetGuestCR3(pVCpu, u64Val);
6345 if (VMMRZCallRing3IsEnabled(pVCpu))
6346 {
6347 PGMUpdateCR3(pVCpu, u64Val);
6348 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6349 }
6350 else
6351 {
6352 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6353 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6354 }
6355 }
6356
6357 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6358 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6359 {
6360 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6361 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6362 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6363 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6364
6365 if (VMMRZCallRing3IsEnabled(pVCpu))
6366 {
6367 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6368 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6369 }
6370 else
6371 {
6372 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6373 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6374 }
6375 }
6376 }
6377
6378 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6379 }
6380
6381 /*
6382 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6383 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6384 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6385 *
6386 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6387 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6388 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6389 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6390 *
6391 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6392 */
6393 if (VMMRZCallRing3IsEnabled(pVCpu))
6394 {
6395 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6396 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6397
6398 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6399 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6400
6401 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6402 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6403 }
6404
6405 return rc;
6406}
6407
6408
6409/**
6410 * Reads a guest segment register from the current VMCS into the guest-CPU
6411 * context.
6412 *
6413 * @returns VBox status code.
6414 * @param pVCpu Pointer to the VMCPU.
6415 * @param idxSel Index of the selector in the VMCS.
6416 * @param idxLimit Index of the segment limit in the VMCS.
6417 * @param idxBase Index of the segment base in the VMCS.
6418 * @param idxAccess Index of the access rights of the segment in the VMCS.
6419 * @param pSelReg Pointer to the segment selector.
6420 *
6421 * @remarks No-long-jump zone!!!
6422 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6423 * macro as that takes care of whether to read from the VMCS cache or
6424 * not.
6425 */
6426DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6427 PCPUMSELREG pSelReg)
6428{
6429 NOREF(pVCpu);
6430
6431 uint32_t u32Val = 0;
6432 int rc = VMXReadVmcs32(idxSel, &u32Val);
6433 AssertRCReturn(rc, rc);
6434 pSelReg->Sel = (uint16_t)u32Val;
6435 pSelReg->ValidSel = (uint16_t)u32Val;
6436 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6437
6438 rc = VMXReadVmcs32(idxLimit, &u32Val);
6439 AssertRCReturn(rc, rc);
6440 pSelReg->u32Limit = u32Val;
6441
6442 uint64_t u64Val = 0;
6443 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6444 AssertRCReturn(rc, rc);
6445 pSelReg->u64Base = u64Val;
6446
6447 rc = VMXReadVmcs32(idxAccess, &u32Val);
6448 AssertRCReturn(rc, rc);
6449 pSelReg->Attr.u = u32Val;
6450
6451 /*
6452 * If VT-x marks the segment as unusable, most other bits remain undefined:
6453 * - For CS the L, D and G bits have meaning.
6454 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6455 * - For the remaining data segments no bits are defined.
6456 *
6457 * The present bit and the unusable bit has been observed to be set at the
6458 * same time (the selector was supposed to be invalid as we started executing
6459 * a V8086 interrupt in ring-0).
6460 *
6461 * What should be important for the rest of the VBox code, is that the P bit is
6462 * cleared. Some of the other VBox code recognizes the unusable bit, but
6463 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6464 * safe side here, we'll strip off P and other bits we don't care about. If
6465 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6466 *
6467 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6468 */
6469 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6470 {
6471 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6472
6473 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6474 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6475 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6476
6477 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6478#ifdef DEBUG_bird
6479 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6480 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6481 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6482#endif
6483 }
6484 return VINF_SUCCESS;
6485}
6486
6487
6488#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6489# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6490 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6491 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6492#else
6493# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6494 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6495 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6496#endif
6497
6498
6499/**
6500 * Saves the guest segment registers from the current VMCS into the guest-CPU
6501 * context.
6502 *
6503 * @returns VBox status code.
6504 * @param pVCpu Pointer to the VMCPU.
6505 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6506 * out-of-sync. Make sure to update the required fields
6507 * before using them.
6508 *
6509 * @remarks No-long-jump zone!!!
6510 */
6511static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6512{
6513 /* Guest segment registers. */
6514 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6515 {
6516 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6517 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6518 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6519 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6520 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6521 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6522 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6523
6524 /* Restore segment attributes for real-on-v86 mode hack. */
6525 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6526 {
6527 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6528 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6529 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6530 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6531 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6532 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6533 }
6534 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6535 }
6536
6537 return VINF_SUCCESS;
6538}
6539
6540
6541/**
6542 * Saves the guest descriptor table registers and task register from the current
6543 * VMCS into the guest-CPU context.
6544 *
6545 * @returns VBox status code.
6546 * @param pVCpu Pointer to the VMCPU.
6547 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6548 * out-of-sync. Make sure to update the required fields
6549 * before using them.
6550 *
6551 * @remarks No-long-jump zone!!!
6552 */
6553static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6554{
6555 int rc = VINF_SUCCESS;
6556
6557 /* Guest LDTR. */
6558 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6559 {
6560 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6561 AssertRCReturn(rc, rc);
6562 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6563 }
6564
6565 /* Guest GDTR. */
6566 uint64_t u64Val = 0;
6567 uint32_t u32Val = 0;
6568 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6569 {
6570 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6571 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6572 pMixedCtx->gdtr.pGdt = u64Val;
6573 pMixedCtx->gdtr.cbGdt = u32Val;
6574 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6575 }
6576
6577 /* Guest IDTR. */
6578 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6579 {
6580 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6581 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6582 pMixedCtx->idtr.pIdt = u64Val;
6583 pMixedCtx->idtr.cbIdt = u32Val;
6584 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6585 }
6586
6587 /* Guest TR. */
6588 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6589 {
6590 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6591 AssertRCReturn(rc, rc);
6592
6593 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6594 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6595 {
6596 rc = VMXLOCAL_READ_SEG(TR, tr);
6597 AssertRCReturn(rc, rc);
6598 }
6599 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6600 }
6601 return rc;
6602}
6603
6604#undef VMXLOCAL_READ_SEG
6605
6606
6607/**
6608 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6609 * context.
6610 *
6611 * @returns VBox status code.
6612 * @param pVCpu Pointer to the VMCPU.
6613 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6614 * out-of-sync. Make sure to update the required fields
6615 * before using them.
6616 *
6617 * @remarks No-long-jump zone!!!
6618 */
6619static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6620{
6621 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6622 {
6623 if (!pVCpu->hm.s.fUsingHyperDR7)
6624 {
6625 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6626 uint32_t u32Val;
6627 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6628 pMixedCtx->dr[7] = u32Val;
6629 }
6630
6631 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6632 }
6633 return VINF_SUCCESS;
6634}
6635
6636
6637/**
6638 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6639 *
6640 * @returns VBox status code.
6641 * @param pVCpu Pointer to the VMCPU.
6642 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6643 * out-of-sync. Make sure to update the required fields
6644 * before using them.
6645 *
6646 * @remarks No-long-jump zone!!!
6647 */
6648static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6649{
6650 NOREF(pMixedCtx);
6651
6652 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6653 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6654 return VINF_SUCCESS;
6655}
6656
6657
6658/**
6659 * Saves the entire guest state from the currently active VMCS into the
6660 * guest-CPU context. This essentially VMREADs all guest-data.
6661 *
6662 * @returns VBox status code.
6663 * @param pVCpu Pointer to the VMCPU.
6664 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6665 * out-of-sync. Make sure to update the required fields
6666 * before using them.
6667 */
6668static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6669{
6670 Assert(pVCpu);
6671 Assert(pMixedCtx);
6672
6673 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6674 return VINF_SUCCESS;
6675
6676 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6677 again on the ring-3 callback path, there is no real need to. */
6678 if (VMMRZCallRing3IsEnabled(pVCpu))
6679 VMMR0LogFlushDisable(pVCpu);
6680 else
6681 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6682 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6683
6684 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6685 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6686
6687 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6688 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6689
6690 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6691 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6692
6693 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6694 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6695
6696 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6697 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6698
6699 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6700 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6701
6702 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6703 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6704
6705 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6706 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6707
6708 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6709 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6710
6711 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6712 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6713
6714 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6715 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6716
6717 if (VMMRZCallRing3IsEnabled(pVCpu))
6718 VMMR0LogFlushEnable(pVCpu);
6719
6720 return rc;
6721}
6722
6723
6724/**
6725 * Check per-VM and per-VCPU force flag actions that require us to go back to
6726 * ring-3 for one reason or another.
6727 *
6728 * @returns VBox status code (information status code included).
6729 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6730 * ring-3.
6731 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6732 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6733 * interrupts)
6734 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6735 * all EMTs to be in ring-3.
6736 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6737 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6738 * to the EM loop.
6739 *
6740 * @param pVM Pointer to the VM.
6741 * @param pVCpu Pointer to the VMCPU.
6742 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6743 * out-of-sync. Make sure to update the required fields
6744 * before using them.
6745 */
6746static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6747{
6748 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6749
6750 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6751 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6752 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6753 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6754 {
6755 /* We need the control registers now, make sure the guest-CPU context is updated. */
6756 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6757 AssertRCReturn(rc3, rc3);
6758
6759 /* Pending HM CR3 sync. */
6760 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6761 {
6762 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6763 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6764 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6765 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6766 }
6767
6768 /* Pending HM PAE PDPEs. */
6769 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6770 {
6771 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6772 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6773 }
6774
6775 /* Pending PGM C3 sync. */
6776 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6777 {
6778 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6779 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6780 if (rc2 != VINF_SUCCESS)
6781 {
6782 AssertRC(rc2);
6783 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6784 return rc2;
6785 }
6786 }
6787
6788 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6789 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6790 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6791 {
6792 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6793 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6794 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6795 return rc2;
6796 }
6797
6798 /* Pending VM request packets, such as hardware interrupts. */
6799 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6800 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6801 {
6802 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6803 return VINF_EM_PENDING_REQUEST;
6804 }
6805
6806 /* Pending PGM pool flushes. */
6807 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6808 {
6809 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6810 return VINF_PGM_POOL_FLUSH_PENDING;
6811 }
6812
6813 /* Pending DMA requests. */
6814 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6815 {
6816 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6817 return VINF_EM_RAW_TO_R3;
6818 }
6819 }
6820
6821 return VINF_SUCCESS;
6822}
6823
6824
6825/**
6826 * Converts any TRPM trap into a pending HM event. This is typically used when
6827 * entering from ring-3 (not longjmp returns).
6828 *
6829 * @param pVCpu Pointer to the VMCPU.
6830 */
6831static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6832{
6833 Assert(TRPMHasTrap(pVCpu));
6834 Assert(!pVCpu->hm.s.Event.fPending);
6835
6836 uint8_t uVector;
6837 TRPMEVENT enmTrpmEvent;
6838 RTGCUINT uErrCode;
6839 RTGCUINTPTR GCPtrFaultAddress;
6840 uint8_t cbInstr;
6841
6842 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6843 AssertRC(rc);
6844
6845 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6846 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6847 if (enmTrpmEvent == TRPM_TRAP)
6848 {
6849 switch (uVector)
6850 {
6851 case X86_XCPT_NMI:
6852 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6853 break;
6854
6855 case X86_XCPT_BP:
6856 case X86_XCPT_OF:
6857 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6858 break;
6859
6860 case X86_XCPT_PF:
6861 case X86_XCPT_DF:
6862 case X86_XCPT_TS:
6863 case X86_XCPT_NP:
6864 case X86_XCPT_SS:
6865 case X86_XCPT_GP:
6866 case X86_XCPT_AC:
6867 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6868 /* no break! */
6869 default:
6870 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6871 break;
6872 }
6873 }
6874 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6875 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6876 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6877 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6878 else
6879 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6880
6881 rc = TRPMResetTrap(pVCpu);
6882 AssertRC(rc);
6883 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6884 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6885
6886 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6887 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6888}
6889
6890
6891/**
6892 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6893 * VT-x to execute any instruction.
6894 *
6895 * @param pvCpu Pointer to the VMCPU.
6896 */
6897static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6898{
6899 Assert(pVCpu->hm.s.Event.fPending);
6900
6901 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6902 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6903 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6904 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6905
6906 /* If a trap was already pending, we did something wrong! */
6907 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6908
6909 TRPMEVENT enmTrapType;
6910 switch (uVectorType)
6911 {
6912 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6913 enmTrapType = TRPM_HARDWARE_INT;
6914 break;
6915
6916 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6917 enmTrapType = TRPM_SOFTWARE_INT;
6918 break;
6919
6920 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6921 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6922 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6923 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6924 enmTrapType = TRPM_TRAP;
6925 break;
6926
6927 default:
6928 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6929 enmTrapType = TRPM_32BIT_HACK;
6930 break;
6931 }
6932
6933 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6934
6935 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6936 AssertRC(rc);
6937
6938 if (fErrorCodeValid)
6939 TRPMSetErrorCode(pVCpu, uErrorCode);
6940
6941 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6942 && uVector == X86_XCPT_PF)
6943 {
6944 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6945 }
6946 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6947 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6948 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6949 {
6950 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6951 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6952 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6953 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6954 }
6955 pVCpu->hm.s.Event.fPending = false;
6956}
6957
6958
6959/**
6960 * Does the necessary state syncing before returning to ring-3 for any reason
6961 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6962 *
6963 * @returns VBox status code.
6964 * @param pVM Pointer to the VM.
6965 * @param pVCpu Pointer to the VMCPU.
6966 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6967 * be out-of-sync. Make sure to update the required
6968 * fields before using them.
6969 * @param fSaveGuestState Whether to save the guest state or not.
6970 *
6971 * @remarks No-long-jmp zone!!!
6972 */
6973static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6974{
6975 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6976 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6977
6978 RTCPUID idCpu = RTMpCpuId();
6979 Log4Func(("HostCpuId=%u\n", idCpu));
6980
6981 /*
6982 * !!! IMPORTANT !!!
6983 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6984 */
6985
6986 /* Save the guest state if necessary. */
6987 if ( fSaveGuestState
6988 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6989 {
6990 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6991 AssertRCReturn(rc, rc);
6992 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6993 }
6994
6995 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6996 if (CPUMIsGuestFPUStateActive(pVCpu))
6997 {
6998 /* We shouldn't reload CR0 without saving it first. */
6999 if (!fSaveGuestState)
7000 {
7001 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7002 AssertRCReturn(rc, rc);
7003 }
7004 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7005 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7006 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7007 }
7008
7009 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7010#ifdef VBOX_STRICT
7011 if (CPUMIsHyperDebugStateActive(pVCpu))
7012 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7013#endif
7014 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7015 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7016 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7017 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7018
7019#if HC_ARCH_BITS == 64
7020 /* Restore host-state bits that VT-x only restores partially. */
7021 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7022 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7023 {
7024 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7025 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7026 }
7027 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7028#endif
7029
7030#if HC_ARCH_BITS == 64
7031 /* Restore the host MSRs as we're leaving VT-x context. */
7032 if ( pVM->hm.s.fAllow64BitGuests
7033 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7034 {
7035 /* We shouldn't reload the guest MSRs without saving it first. */
7036 if (!fSaveGuestState)
7037 {
7038 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7039 AssertRCReturn(rc, rc);
7040 }
7041 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7042 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7043 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
7044 }
7045#endif
7046
7047 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7048 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7049
7050 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7051 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7052 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7053 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7054 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7055 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7056 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7057 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7058
7059 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7060
7061 /** @todo This partially defeats the purpose of having preemption hooks.
7062 * The problem is, deregistering the hooks should be moved to a place that
7063 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7064 * context.
7065 */
7066 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7067 {
7068 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7069 AssertRCReturn(rc, rc);
7070
7071 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7072 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7073 }
7074 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7075 NOREF(idCpu);
7076
7077 return VINF_SUCCESS;
7078}
7079
7080
7081/**
7082 * Leaves the VT-x session.
7083 *
7084 * @returns VBox status code.
7085 * @param pVM Pointer to the VM.
7086 * @param pVCpu Pointer to the VMCPU.
7087 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7088 * out-of-sync. Make sure to update the required fields
7089 * before using them.
7090 *
7091 * @remarks No-long-jmp zone!!!
7092 */
7093DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7094{
7095 HM_DISABLE_PREEMPT_IF_NEEDED();
7096 HMVMX_ASSERT_CPU_SAFE();
7097 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7098 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7099
7100 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7101 and done this from the VMXR0ThreadCtxCallback(). */
7102 if (!pVCpu->hm.s.fLeaveDone)
7103 {
7104 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7105 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7106 pVCpu->hm.s.fLeaveDone = true;
7107 }
7108 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7109
7110 /*
7111 * !!! IMPORTANT !!!
7112 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7113 */
7114
7115 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7116 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7117 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7118 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7119 VMMR0ThreadCtxHooksDeregister(pVCpu);
7120
7121 /* Leave HM context. This takes care of local init (term). */
7122 int rc = HMR0LeaveCpu(pVCpu);
7123
7124 HM_RESTORE_PREEMPT_IF_NEEDED();
7125
7126 return rc;
7127}
7128
7129
7130/**
7131 * Does the necessary state syncing before doing a longjmp to ring-3.
7132 *
7133 * @returns VBox status code.
7134 * @param pVM Pointer to the VM.
7135 * @param pVCpu Pointer to the VMCPU.
7136 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7137 * out-of-sync. Make sure to update the required fields
7138 * before using them.
7139 *
7140 * @remarks No-long-jmp zone!!!
7141 */
7142DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7143{
7144 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7145}
7146
7147
7148/**
7149 * Take necessary actions before going back to ring-3.
7150 *
7151 * An action requires us to go back to ring-3. This function does the necessary
7152 * steps before we can safely return to ring-3. This is not the same as longjmps
7153 * to ring-3, this is voluntary and prepares the guest so it may continue
7154 * executing outside HM (recompiler/IEM).
7155 *
7156 * @returns VBox status code.
7157 * @param pVM Pointer to the VM.
7158 * @param pVCpu Pointer to the VMCPU.
7159 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7160 * out-of-sync. Make sure to update the required fields
7161 * before using them.
7162 * @param rcExit The reason for exiting to ring-3. Can be
7163 * VINF_VMM_UNKNOWN_RING3_CALL.
7164 */
7165static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7166{
7167 Assert(pVM);
7168 Assert(pVCpu);
7169 Assert(pMixedCtx);
7170 HMVMX_ASSERT_PREEMPT_SAFE();
7171
7172 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7173 {
7174 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7175 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7176 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7177 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7178 }
7179
7180 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7181 VMMRZCallRing3Disable(pVCpu);
7182 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7183
7184 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7185 if (pVCpu->hm.s.Event.fPending)
7186 {
7187 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7188 Assert(!pVCpu->hm.s.Event.fPending);
7189 }
7190
7191 /* Save guest state and restore host state bits. */
7192 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7193 AssertRCReturn(rc, rc);
7194 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7195
7196 /* Sync recompiler state. */
7197 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7198 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7199 | CPUM_CHANGED_LDTR
7200 | CPUM_CHANGED_GDTR
7201 | CPUM_CHANGED_IDTR
7202 | CPUM_CHANGED_TR
7203 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7204 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7205 if ( pVM->hm.s.fNestedPaging
7206 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7207 {
7208 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7209 }
7210
7211 Assert(!pVCpu->hm.s.fClearTrapFlag);
7212
7213 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7214 if (rcExit != VINF_EM_RAW_INTERRUPT)
7215 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7216
7217 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7218
7219 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7220 VMMRZCallRing3RemoveNotification(pVCpu);
7221 VMMRZCallRing3Enable(pVCpu);
7222
7223 return rc;
7224}
7225
7226
7227/**
7228 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7229 * longjump to ring-3 and possibly get preempted.
7230 *
7231 * @returns VBox status code.
7232 * @param pVCpu Pointer to the VMCPU.
7233 * @param enmOperation The operation causing the ring-3 longjump.
7234 * @param pvUser Opaque pointer to the guest-CPU context. The data
7235 * may be out-of-sync. Make sure to update the required
7236 * fields before using them.
7237 */
7238DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7239{
7240 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7241 {
7242 /*
7243 * !!! IMPORTANT !!!
7244 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7245 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7246 */
7247 VMMRZCallRing3RemoveNotification(pVCpu);
7248 VMMRZCallRing3Disable(pVCpu);
7249 HM_DISABLE_PREEMPT_IF_NEEDED();
7250
7251 PVM pVM = pVCpu->CTX_SUFF(pVM);
7252 if (CPUMIsGuestFPUStateActive(pVCpu))
7253 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7254
7255 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7256
7257#if HC_ARCH_BITS == 64
7258 /* Restore host-state bits that VT-x only restores partially. */
7259 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7260 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7261 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7262 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7263
7264 /* Restore the host MSRs as we're leaving VT-x context. */
7265 if ( pVM->hm.s.fAllow64BitGuests
7266 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7267 {
7268 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7269 }
7270#endif
7271 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7272 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7273 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7274 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7275 {
7276 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7277 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7278 }
7279
7280 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7281 VMMR0ThreadCtxHooksDeregister(pVCpu);
7282
7283 HMR0LeaveCpu(pVCpu);
7284 HM_RESTORE_PREEMPT_IF_NEEDED();
7285 return VINF_SUCCESS;
7286 }
7287
7288 Assert(pVCpu);
7289 Assert(pvUser);
7290 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7291 HMVMX_ASSERT_PREEMPT_SAFE();
7292
7293 VMMRZCallRing3Disable(pVCpu);
7294 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7295
7296 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7297 enmOperation));
7298
7299 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7300 AssertRCReturn(rc, rc);
7301
7302 VMMRZCallRing3Enable(pVCpu);
7303 return VINF_SUCCESS;
7304}
7305
7306
7307/**
7308 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7309 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7310 *
7311 * @param pVCpu Pointer to the VMCPU.
7312 */
7313DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7314{
7315 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7316 {
7317 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7318 {
7319 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7320 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7321 AssertRC(rc);
7322 Log4(("Setup interrupt-window exiting\n"));
7323 }
7324 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7325}
7326
7327
7328/**
7329 * Clears the interrupt-window exiting control in the VMCS.
7330 *
7331 * @param pVCpu Pointer to the VMCPU.
7332 */
7333DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7334{
7335 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7336 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7337 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7338 AssertRC(rc);
7339 Log4(("Cleared interrupt-window exiting\n"));
7340}
7341
7342
7343/**
7344 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7345 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7346 *
7347 * @param pVCpu Pointer to the VMCPU.
7348 */
7349DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7350{
7351 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7352 {
7353 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7354 {
7355 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7356 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7357 AssertRC(rc);
7358 Log4(("Setup NMI-window exiting\n"));
7359 }
7360 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7361}
7362
7363
7364/**
7365 * Clears the NMI-window exiting control in the VMCS.
7366 *
7367 * @param pVCpu Pointer to the VMCPU.
7368 */
7369DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7370{
7371 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7372 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7373 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7374 AssertRC(rc);
7375 Log4(("Cleared NMI-window exiting\n"));
7376}
7377
7378
7379/**
7380 * Evaluates the event to be delivered to the guest and sets it as the pending
7381 * event.
7382 *
7383 * @param pVCpu Pointer to the VMCPU.
7384 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7385 * out-of-sync. Make sure to update the required fields
7386 * before using them.
7387 */
7388static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7389{
7390 Assert(!pVCpu->hm.s.Event.fPending);
7391
7392 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7393 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7394 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7395 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7396 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7397
7398 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7399 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7400 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7401 Assert(!TRPMHasTrap(pVCpu));
7402
7403 /*
7404 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7405 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7406 */
7407 /** @todo SMI. SMIs take priority over NMIs. */
7408 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7409 {
7410 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7411 if ( !fBlockNmi
7412 && !fBlockSti
7413 && !fBlockMovSS)
7414 {
7415 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7416 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7417 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7418
7419 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7420 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7421 }
7422 else
7423 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7424 }
7425 /*
7426 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7427 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7428 */
7429 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7430 && !pVCpu->hm.s.fSingleInstruction)
7431 {
7432 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7433 AssertRC(rc);
7434 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7435 if ( !fBlockInt
7436 && !fBlockSti
7437 && !fBlockMovSS)
7438 {
7439 uint8_t u8Interrupt;
7440 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7441 if (RT_SUCCESS(rc))
7442 {
7443 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7444 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7445 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7446
7447 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7448 }
7449 else
7450 {
7451 /** @todo Does this actually happen? If not turn it into an assertion. */
7452 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7453 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7454 }
7455 }
7456 else
7457 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7458 }
7459}
7460
7461
7462/**
7463 * Sets a pending-debug exception to be delivered to the guest if the guest is
7464 * single-stepping.
7465 *
7466 * @param pVCpu Pointer to the VMCPU.
7467 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7468 * out-of-sync. Make sure to update the required fields
7469 * before using them.
7470 */
7471DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7472{
7473 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7474 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7475 {
7476 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7477 AssertRC(rc);
7478 }
7479}
7480
7481
7482/**
7483 * Injects any pending events into the guest if the guest is in a state to
7484 * receive them.
7485 *
7486 * @returns VBox status code (informational status codes included).
7487 * @param pVCpu Pointer to the VMCPU.
7488 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7489 * out-of-sync. Make sure to update the required fields
7490 * before using them.
7491 */
7492static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7493{
7494 HMVMX_ASSERT_PREEMPT_SAFE();
7495 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7496
7497 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7498 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7499 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7500 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7501
7502 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7503 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7504 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7505 Assert(!TRPMHasTrap(pVCpu));
7506
7507 int rc = VINF_SUCCESS;
7508 if (pVCpu->hm.s.Event.fPending)
7509 {
7510 /*
7511 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7512 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7513 * ended up enabling interrupts outside VT-x.
7514 */
7515 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7516 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7517 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7518 {
7519 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7520 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7521 }
7522
7523#ifdef VBOX_STRICT
7524 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7525 {
7526 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7527 Assert(!fBlockInt);
7528 Assert(!fBlockSti);
7529 Assert(!fBlockMovSS);
7530 }
7531 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7532 {
7533 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7534 Assert(!fBlockSti);
7535 Assert(!fBlockMovSS);
7536 Assert(!fBlockNmi);
7537 }
7538#endif
7539 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7540 (uint8_t)uIntType));
7541 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7542 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7543 AssertRCReturn(rc, rc);
7544
7545 /* Update the interruptibility-state as it could have been changed by
7546 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7547 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7548 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7549
7550#ifdef VBOX_WITH_STATISTICS
7551 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7552 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7553 else
7554 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7555#endif
7556 }
7557
7558 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7559 if ( fBlockSti
7560 || fBlockMovSS)
7561 {
7562 if ( !pVCpu->hm.s.fSingleInstruction
7563 && !DBGFIsStepping(pVCpu))
7564 {
7565 /*
7566 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7567 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7568 * See Intel spec. 27.3.4 "Saving Non-Register State".
7569 */
7570 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7571 AssertRCReturn(rc2, rc2);
7572 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7573 }
7574 else if (pMixedCtx->eflags.Bits.u1TF)
7575 {
7576 /*
7577 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7578 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7579 */
7580 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7581 uIntrState = 0;
7582 }
7583 }
7584
7585 /*
7586 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7587 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7588 */
7589 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7590 AssertRC(rc2);
7591
7592 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7593 NOREF(fBlockMovSS); NOREF(fBlockSti);
7594 return rc;
7595}
7596
7597
7598/**
7599 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7600 *
7601 * @param pVCpu Pointer to the VMCPU.
7602 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7603 * out-of-sync. Make sure to update the required fields
7604 * before using them.
7605 */
7606DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7607{
7608 NOREF(pMixedCtx);
7609 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7610 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7611}
7612
7613
7614/**
7615 * Injects a double-fault (#DF) exception into the VM.
7616 *
7617 * @returns VBox status code (informational status code included).
7618 * @param pVCpu Pointer to the VMCPU.
7619 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7620 * out-of-sync. Make sure to update the required fields
7621 * before using them.
7622 */
7623DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7624{
7625 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7626 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7627 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7628 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7629 puIntrState);
7630}
7631
7632
7633/**
7634 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7635 *
7636 * @param pVCpu Pointer to the VMCPU.
7637 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7638 * out-of-sync. Make sure to update the required fields
7639 * before using them.
7640 */
7641DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7642{
7643 NOREF(pMixedCtx);
7644 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7645 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7646 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7647}
7648
7649
7650/**
7651 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7652 *
7653 * @param pVCpu Pointer to the VMCPU.
7654 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7655 * out-of-sync. Make sure to update the required fields
7656 * before using them.
7657 * @param cbInstr The value of RIP that is to be pushed on the guest
7658 * stack.
7659 */
7660DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7661{
7662 NOREF(pMixedCtx);
7663 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7664 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7665 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7666}
7667
7668
7669/**
7670 * Injects a general-protection (#GP) fault into the VM.
7671 *
7672 * @returns VBox status code (informational status code included).
7673 * @param pVCpu Pointer to the VMCPU.
7674 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7675 * out-of-sync. Make sure to update the required fields
7676 * before using them.
7677 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7678 * mode, i.e. in real-mode it's not valid).
7679 * @param u32ErrorCode The error code associated with the #GP.
7680 */
7681DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7682 uint32_t *puIntrState)
7683{
7684 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7685 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7686 if (fErrorCodeValid)
7687 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7688 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7689 puIntrState);
7690}
7691
7692
7693/**
7694 * Sets a general-protection (#GP) exception as pending-for-injection into the
7695 * VM.
7696 *
7697 * @param pVCpu Pointer to the VMCPU.
7698 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7699 * out-of-sync. Make sure to update the required fields
7700 * before using them.
7701 * @param u32ErrorCode The error code associated with the #GP.
7702 */
7703DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7704{
7705 NOREF(pMixedCtx);
7706 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7707 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7708 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7709 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7710}
7711
7712
7713/**
7714 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7715 *
7716 * @param pVCpu Pointer to the VMCPU.
7717 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7718 * out-of-sync. Make sure to update the required fields
7719 * before using them.
7720 * @param uVector The software interrupt vector number.
7721 * @param cbInstr The value of RIP that is to be pushed on the guest
7722 * stack.
7723 */
7724DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7725{
7726 NOREF(pMixedCtx);
7727 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7728 if ( uVector == X86_XCPT_BP
7729 || uVector == X86_XCPT_OF)
7730 {
7731 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7732 }
7733 else
7734 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7735 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7736}
7737
7738
7739/**
7740 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7741 * stack.
7742 *
7743 * @returns VBox status code (information status code included).
7744 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7745 * @param pVM Pointer to the VM.
7746 * @param pMixedCtx Pointer to the guest-CPU context.
7747 * @param uValue The value to push to the guest stack.
7748 */
7749DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7750{
7751 /*
7752 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7753 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7754 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7755 */
7756 if (pMixedCtx->sp == 1)
7757 return VINF_EM_RESET;
7758 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7759 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7760 AssertRCReturn(rc, rc);
7761 return rc;
7762}
7763
7764
7765/**
7766 * Injects an event into the guest upon VM-entry by updating the relevant fields
7767 * in the VM-entry area in the VMCS.
7768 *
7769 * @returns VBox status code (informational error codes included).
7770 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7771 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7772 *
7773 * @param pVCpu Pointer to the VMCPU.
7774 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7775 * be out-of-sync. Make sure to update the required
7776 * fields before using them.
7777 * @param u64IntInfo The VM-entry interruption-information field.
7778 * @param cbInstr The VM-entry instruction length in bytes (for
7779 * software interrupts, exceptions and privileged
7780 * software exceptions).
7781 * @param u32ErrCode The VM-entry exception error code.
7782 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7783 * @param puIntrState Pointer to the current guest interruptibility-state.
7784 * This interruptibility-state will be updated if
7785 * necessary. This cannot not be NULL.
7786 *
7787 * @remarks Requires CR0!
7788 * @remarks No-long-jump zone!!!
7789 */
7790static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7791 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7792{
7793 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7794 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7795 Assert(puIntrState);
7796 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7797
7798 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7799 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7800
7801#ifdef VBOX_STRICT
7802 /* Validate the error-code-valid bit for hardware exceptions. */
7803 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7804 {
7805 switch (uVector)
7806 {
7807 case X86_XCPT_PF:
7808 case X86_XCPT_DF:
7809 case X86_XCPT_TS:
7810 case X86_XCPT_NP:
7811 case X86_XCPT_SS:
7812 case X86_XCPT_GP:
7813 case X86_XCPT_AC:
7814 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7815 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7816 /* fallthru */
7817 default:
7818 break;
7819 }
7820 }
7821#endif
7822
7823 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7824 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7825 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7826
7827 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7828
7829 /* We require CR0 to check if the guest is in real-mode. */
7830 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7831 AssertRCReturn(rc, rc);
7832
7833 /*
7834 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7835 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7836 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7837 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7838 */
7839 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7840 {
7841 PVM pVM = pVCpu->CTX_SUFF(pVM);
7842 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7843 {
7844 Assert(PDMVmmDevHeapIsEnabled(pVM));
7845 Assert(pVM->hm.s.vmx.pRealModeTSS);
7846
7847 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7848 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7849 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7850 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7851 AssertRCReturn(rc, rc);
7852 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7853
7854 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7855 size_t const cbIdtEntry = sizeof(X86IDTR16);
7856 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7857 {
7858 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7859 if (uVector == X86_XCPT_DF)
7860 return VINF_EM_RESET;
7861 else if (uVector == X86_XCPT_GP)
7862 {
7863 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7864 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7865 }
7866
7867 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7868 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7869 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7870 }
7871
7872 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7873 uint16_t uGuestIp = pMixedCtx->ip;
7874 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7875 {
7876 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7877 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7878 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7879 }
7880 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7881 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7882
7883 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7884 X86IDTR16 IdtEntry;
7885 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7886 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7887 AssertRCReturn(rc, rc);
7888
7889 /* Construct the stack frame for the interrupt/exception handler. */
7890 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7891 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7892 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7893 AssertRCReturn(rc, rc);
7894
7895 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7896 if (rc == VINF_SUCCESS)
7897 {
7898 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7899 pMixedCtx->rip = IdtEntry.offSel;
7900 pMixedCtx->cs.Sel = IdtEntry.uSel;
7901 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7902 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7903 && uVector == X86_XCPT_PF)
7904 {
7905 pMixedCtx->cr2 = GCPtrFaultAddress;
7906 }
7907
7908 /* If any other guest-state bits are changed here, make sure to update
7909 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7910 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7911 | HM_CHANGED_GUEST_RIP
7912 | HM_CHANGED_GUEST_RFLAGS
7913 | HM_CHANGED_GUEST_RSP);
7914
7915 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7916 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7917 {
7918 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7919 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7920 Log4(("Clearing inhibition due to STI.\n"));
7921 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7922 }
7923 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7924
7925 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7926 it, if we are returning to ring-3 before executing guest code. */
7927 pVCpu->hm.s.Event.fPending = false;
7928 }
7929 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7930 return rc;
7931 }
7932 else
7933 {
7934 /*
7935 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7936 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7937 */
7938 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7939 }
7940 }
7941
7942 /* Validate. */
7943 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7944 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7945 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7946
7947 /* Inject. */
7948 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7949 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7950 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7951 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7952
7953 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7954 && uVector == X86_XCPT_PF)
7955 {
7956 pMixedCtx->cr2 = GCPtrFaultAddress;
7957 }
7958
7959 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7960 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7961
7962 AssertRCReturn(rc, rc);
7963 return rc;
7964}
7965
7966
7967/**
7968 * Clears the interrupt-window exiting control in the VMCS and if necessary
7969 * clears the current event in the VMCS as well.
7970 *
7971 * @returns VBox status code.
7972 * @param pVCpu Pointer to the VMCPU.
7973 *
7974 * @remarks Use this function only to clear events that have not yet been
7975 * delivered to the guest but are injected in the VMCS!
7976 * @remarks No-long-jump zone!!!
7977 */
7978static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7979{
7980 int rc;
7981 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7982
7983 /* Clear interrupt-window exiting control. */
7984 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7985 {
7986 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7987 Assert(!pVCpu->hm.s.Event.fPending);
7988 }
7989
7990 /* Clear NMI-window exiting control. */
7991 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7992 {
7993 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7994 Assert(!pVCpu->hm.s.Event.fPending);
7995 }
7996
7997 if (!pVCpu->hm.s.Event.fPending)
7998 return;
7999
8000#ifdef VBOX_STRICT
8001 uint32_t u32EntryInfo;
8002 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8003 AssertRC(rc);
8004 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8005#endif
8006
8007 /* Clear the entry-interruption field (including the valid bit). */
8008 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8009 AssertRC(rc);
8010
8011 /* Clear the pending debug exception field. */
8012 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8013 AssertRC(rc);
8014
8015 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8016 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8017}
8018
8019
8020/**
8021 * Enters the VT-x session.
8022 *
8023 * @returns VBox status code.
8024 * @param pVM Pointer to the VM.
8025 * @param pVCpu Pointer to the VMCPU.
8026 * @param pCpu Pointer to the CPU info struct.
8027 */
8028VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8029{
8030 AssertPtr(pVM);
8031 AssertPtr(pVCpu);
8032 Assert(pVM->hm.s.vmx.fSupported);
8033 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8034 NOREF(pCpu); NOREF(pVM);
8035
8036 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8037 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8038
8039#ifdef VBOX_STRICT
8040 /* Make sure we're in VMX root mode. */
8041 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8042 if (!(u32HostCR4 & X86_CR4_VMXE))
8043 {
8044 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8045 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8046 }
8047#endif
8048
8049 /*
8050 * Load the VCPU's VMCS as the current (and active) one.
8051 */
8052 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8053 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8054 if (RT_FAILURE(rc))
8055 return rc;
8056
8057 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8058 pVCpu->hm.s.fLeaveDone = false;
8059 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8060
8061 return VINF_SUCCESS;
8062}
8063
8064
8065/**
8066 * The thread-context callback (only on platforms which support it).
8067 *
8068 * @param enmEvent The thread-context event.
8069 * @param pVCpu Pointer to the VMCPU.
8070 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8071 * @thread EMT(pVCpu)
8072 */
8073VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8074{
8075 NOREF(fGlobalInit);
8076
8077 switch (enmEvent)
8078 {
8079 case RTTHREADCTXEVENT_PREEMPTING:
8080 {
8081 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8082 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8083 VMCPU_ASSERT_EMT(pVCpu);
8084
8085 PVM pVM = pVCpu->CTX_SUFF(pVM);
8086 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8087
8088 /* No longjmps (logger flushes, locks) in this fragile context. */
8089 VMMRZCallRing3Disable(pVCpu);
8090 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8091
8092 /*
8093 * Restore host-state (FPU, debug etc.)
8094 */
8095 if (!pVCpu->hm.s.fLeaveDone)
8096 {
8097 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8098 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8099 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8100 pVCpu->hm.s.fLeaveDone = true;
8101 }
8102
8103 /* Leave HM context, takes care of local init (term). */
8104 int rc = HMR0LeaveCpu(pVCpu);
8105 AssertRC(rc); NOREF(rc);
8106
8107 /* Restore longjmp state. */
8108 VMMRZCallRing3Enable(pVCpu);
8109 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8110 break;
8111 }
8112
8113 case RTTHREADCTXEVENT_RESUMED:
8114 {
8115 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8116 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8117 VMCPU_ASSERT_EMT(pVCpu);
8118
8119 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8120 VMMRZCallRing3Disable(pVCpu);
8121 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8122
8123 /* Initialize the bare minimum state required for HM. This takes care of
8124 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8125 int rc = HMR0EnterCpu(pVCpu);
8126 AssertRC(rc);
8127 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8128
8129 /* Load the active VMCS as the current one. */
8130 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8131 {
8132 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8133 AssertRC(rc); NOREF(rc);
8134 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8135 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8136 }
8137 pVCpu->hm.s.fLeaveDone = false;
8138
8139 /* Restore longjmp state. */
8140 VMMRZCallRing3Enable(pVCpu);
8141 break;
8142 }
8143
8144 default:
8145 break;
8146 }
8147}
8148
8149
8150/**
8151 * Saves the host state in the VMCS host-state.
8152 * Sets up the VM-exit MSR-load area.
8153 *
8154 * The CPU state will be loaded from these fields on every successful VM-exit.
8155 *
8156 * @returns VBox status code.
8157 * @param pVM Pointer to the VM.
8158 * @param pVCpu Pointer to the VMCPU.
8159 *
8160 * @remarks No-long-jump zone!!!
8161 */
8162static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8163{
8164 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8165
8166 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8167 return VINF_SUCCESS;
8168
8169 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8170 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8171
8172 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8173 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8174
8175 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8176 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8177
8178 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8179 return rc;
8180}
8181
8182
8183/**
8184 * Saves the host state in the VMCS host-state.
8185 *
8186 * @returns VBox status code.
8187 * @param pVM Pointer to the VM.
8188 * @param pVCpu Pointer to the VMCPU.
8189 *
8190 * @remarks No-long-jump zone!!!
8191 */
8192VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8193{
8194 AssertPtr(pVM);
8195 AssertPtr(pVCpu);
8196
8197 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8198
8199 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8200 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8201 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8202 return hmR0VmxSaveHostState(pVM, pVCpu);
8203}
8204
8205
8206/**
8207 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8208 * loaded from these fields on every successful VM-entry.
8209 *
8210 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8211 * Sets up the VM-entry controls.
8212 * Sets up the appropriate VMX non-root function to execute guest code based on
8213 * the guest CPU mode.
8214 *
8215 * @returns VBox status code.
8216 * @param pVM Pointer to the VM.
8217 * @param pVCpu Pointer to the VMCPU.
8218 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8219 * out-of-sync. Make sure to update the required fields
8220 * before using them.
8221 *
8222 * @remarks No-long-jump zone!!!
8223 */
8224static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8225{
8226 AssertPtr(pVM);
8227 AssertPtr(pVCpu);
8228 AssertPtr(pMixedCtx);
8229 HMVMX_ASSERT_PREEMPT_SAFE();
8230
8231#ifdef LOG_ENABLED
8232 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8233 * probably not initialized yet? Anyway this will do for now.
8234 *
8235 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8236 * interface and disable ring-3 calls when thread-context hooks are not
8237 * available. */
8238 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8239 VMMR0LogFlushDisable(pVCpu);
8240#endif
8241
8242 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8243
8244 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8245
8246 /* Determine real-on-v86 mode. */
8247 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8248 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8249 && CPUMIsGuestInRealModeEx(pMixedCtx))
8250 {
8251 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8252 }
8253
8254 /*
8255 * Load the guest-state into the VMCS.
8256 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8257 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8258 */
8259 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8260 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8261
8262 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8263 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8264 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8265
8266 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8267 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8268 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8269
8270 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8271 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8272
8273 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8274 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8275
8276 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8277 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8278 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8279
8280 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8281 determine we don't have to swap EFER after all. */
8282 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8283 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8284
8285 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8286 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8287
8288 /*
8289 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8290 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8291 */
8292 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8293 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8294
8295 /* Clear any unused and reserved bits. */
8296 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8297
8298#ifdef LOG_ENABLED
8299 /* Only reenable log-flushing if the caller has it enabled. */
8300 if (!fCallerDisabledLogFlush)
8301 VMMR0LogFlushEnable(pVCpu);
8302#endif
8303
8304 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8305 return rc;
8306}
8307
8308
8309/**
8310 * Loads the state shared between the host and guest into the VMCS.
8311 *
8312 * @param pVM Pointer to the VM.
8313 * @param pVCpu Pointer to the VMCPU.
8314 * @param pCtx Pointer to the guest-CPU context.
8315 *
8316 * @remarks No-long-jump zone!!!
8317 */
8318static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8319{
8320 NOREF(pVM);
8321
8322 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8323 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8324
8325 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8326 {
8327 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8328 AssertRC(rc);
8329 }
8330
8331 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8332 {
8333 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8334 AssertRC(rc);
8335
8336 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8337 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8338 {
8339 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8340 AssertRC(rc);
8341 }
8342 }
8343
8344 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8345 {
8346#if HC_ARCH_BITS == 64
8347 if (pVM->hm.s.fAllow64BitGuests)
8348 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8349#endif
8350 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8351 }
8352
8353 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8354 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8355}
8356
8357
8358/**
8359 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8360 *
8361 * @param pVM Pointer to the VM.
8362 * @param pVCpu Pointer to the VMCPU.
8363 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8364 * out-of-sync. Make sure to update the required fields
8365 * before using them.
8366 */
8367DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8368{
8369 HMVMX_ASSERT_PREEMPT_SAFE();
8370
8371 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8372#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8373 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8374#endif
8375
8376 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8377 {
8378 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8379 AssertRC(rc);
8380 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8381 }
8382 else if (HMCPU_CF_VALUE(pVCpu))
8383 {
8384 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8385 AssertRC(rc);
8386 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8387 }
8388
8389 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8390 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8391 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8392 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8393}
8394
8395
8396/**
8397 * Does the preparations before executing guest code in VT-x.
8398 *
8399 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8400 * recompiler. We must be cautious what we do here regarding committing
8401 * guest-state information into the VMCS assuming we assuredly execute the
8402 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8403 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8404 * so that the recompiler can (and should) use them when it resumes guest
8405 * execution. Otherwise such operations must be done when we can no longer
8406 * exit to ring-3.
8407 *
8408 * @returns Strict VBox status code.
8409 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8410 * have been disabled.
8411 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8412 * double-fault into the guest.
8413 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8414 *
8415 * @param pVM Pointer to the VM.
8416 * @param pVCpu Pointer to the VMCPU.
8417 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8418 * out-of-sync. Make sure to update the required fields
8419 * before using them.
8420 * @param pVmxTransient Pointer to the VMX transient structure.
8421 */
8422static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8423{
8424 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8425
8426#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8427 PGMRZDynMapFlushAutoSet(pVCpu);
8428#endif
8429
8430 /* Check force flag actions that might require us to go back to ring-3. */
8431 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8432 if (rc != VINF_SUCCESS)
8433 return rc;
8434
8435#ifndef IEM_VERIFICATION_MODE_FULL
8436 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8437 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8438 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8439 {
8440 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8441 RTGCPHYS GCPhysApicBase;
8442 GCPhysApicBase = pMixedCtx->msrApicBase;
8443 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8444
8445 /* Unalias any existing mapping. */
8446 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8447 AssertRCReturn(rc, rc);
8448
8449 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8450 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8451 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8452 AssertRCReturn(rc, rc);
8453
8454 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8455 }
8456#endif /* !IEM_VERIFICATION_MODE_FULL */
8457
8458 if (TRPMHasTrap(pVCpu))
8459 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8460 else if (!pVCpu->hm.s.Event.fPending)
8461 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8462
8463 /*
8464 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8465 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8466 */
8467 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8468 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8469 {
8470 Assert(rc == VINF_EM_RESET);
8471 return rc;
8472 }
8473
8474 /*
8475 * Load the guest state bits, we can handle longjmps/getting preempted here.
8476 *
8477 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8478 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8479 * Hence, this needs to be done -after- injection of events.
8480 */
8481 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8482
8483 /*
8484 * No longjmps to ring-3 from this point on!!!
8485 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8486 * This also disables flushing of the R0-logger instance (if any).
8487 */
8488 VMMRZCallRing3Disable(pVCpu);
8489
8490 /*
8491 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8492 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8493 *
8494 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8495 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8496 *
8497 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8498 * executing guest code.
8499 */
8500 pVmxTransient->uEflags = ASMIntDisableFlags();
8501 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8502 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8503 {
8504 hmR0VmxClearEventVmcs(pVCpu);
8505 ASMSetFlags(pVmxTransient->uEflags);
8506 VMMRZCallRing3Enable(pVCpu);
8507 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8508 return VINF_EM_RAW_TO_R3;
8509 }
8510
8511 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8512 {
8513 hmR0VmxClearEventVmcs(pVCpu);
8514 ASMSetFlags(pVmxTransient->uEflags);
8515 VMMRZCallRing3Enable(pVCpu);
8516 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8517 return VINF_EM_RAW_INTERRUPT;
8518 }
8519
8520 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8521 pVCpu->hm.s.Event.fPending = false;
8522
8523 return VINF_SUCCESS;
8524}
8525
8526
8527/**
8528 * Prepares to run guest code in VT-x and we've committed to doing so. This
8529 * means there is no backing out to ring-3 or anywhere else at this
8530 * point.
8531 *
8532 * @param pVM Pointer to the VM.
8533 * @param pVCpu Pointer to the VMCPU.
8534 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8535 * out-of-sync. Make sure to update the required fields
8536 * before using them.
8537 * @param pVmxTransient Pointer to the VMX transient structure.
8538 *
8539 * @remarks Called with preemption disabled.
8540 * @remarks No-long-jump zone!!!
8541 */
8542static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8543{
8544 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8545 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8546 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8547
8548 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8549 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8550
8551#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8552 if (!CPUMIsGuestFPUStateActive(pVCpu))
8553 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8554 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8555#endif
8556
8557 if ( pVCpu->hm.s.fUseGuestFpu
8558 && !CPUMIsGuestFPUStateActive(pVCpu))
8559 {
8560 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8561 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8562 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8563 }
8564
8565 /*
8566 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8567 */
8568 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8569 && pVCpu->hm.s.vmx.cMsrs > 0)
8570 {
8571 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8572 }
8573
8574 /*
8575 * Load the host state bits as we may've been preempted (only happens when
8576 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8577 */
8578 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8579 {
8580 /* This ASSUMES that pfnStartVM has been set up already. */
8581 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8582 AssertRC(rc);
8583 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8584 }
8585 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8586
8587 /*
8588 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8589 */
8590 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8591 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8592 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8593
8594 /* Store status of the shared guest-host state at the time of VM-entry. */
8595#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8596 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8597 {
8598 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8599 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8600 }
8601 else
8602#endif
8603 {
8604 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8605 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8606 }
8607 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8608
8609 /*
8610 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8611 */
8612 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8613 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8614
8615 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8616 RTCPUID idCurrentCpu = pCpu->idCpu;
8617 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8618 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8619 {
8620 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8621 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8622 }
8623
8624 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8625 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8626 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8627 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8628
8629 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8630
8631 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8632 to start executing. */
8633
8634 /*
8635 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8636 */
8637 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8638 {
8639 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8640 {
8641 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8642 AssertRC(rc2);
8643 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8644 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8645 true /* fUpdateHostMsr */);
8646 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8647 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8648 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8649 }
8650 else
8651 {
8652 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8653 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8654 }
8655 }
8656
8657#ifdef VBOX_STRICT
8658 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8659 hmR0VmxCheckHostEferMsr(pVCpu);
8660 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8661#endif
8662#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8663 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8664 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8665 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8666#endif
8667}
8668
8669
8670/**
8671 * Performs some essential restoration of state after running guest code in
8672 * VT-x.
8673 *
8674 * @param pVM Pointer to the VM.
8675 * @param pVCpu Pointer to the VMCPU.
8676 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8677 * out-of-sync. Make sure to update the required fields
8678 * before using them.
8679 * @param pVmxTransient Pointer to the VMX transient structure.
8680 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8681 *
8682 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8683 *
8684 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8685 * unconditionally when it is safe to do so.
8686 */
8687static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8688{
8689 NOREF(pVM);
8690
8691 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8692
8693 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8694 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8695 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8696 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8697 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8698
8699 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8700 {
8701 /** @todo Find a way to fix hardcoding a guestimate. */
8702 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8703 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8704 }
8705
8706 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8707 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8708 Assert(!(ASMGetFlags() & X86_EFL_IF));
8709 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8710
8711#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8712 if (CPUMIsGuestFPUStateActive(pVCpu))
8713 {
8714 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8715 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8716 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8717 }
8718#endif
8719
8720#if HC_ARCH_BITS == 64
8721 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8722#endif
8723 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8724#ifdef VBOX_STRICT
8725 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8726#endif
8727 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8728 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8729
8730 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8731 uint32_t uExitReason;
8732 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8733 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8734 AssertRC(rc);
8735 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8736 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8737
8738 /* Update the VM-exit history array. */
8739 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8740
8741 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8742 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8743 {
8744 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8745 pVmxTransient->fVMEntryFailed));
8746 return;
8747 }
8748
8749 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8750 {
8751 /** @todo We can optimize this by only syncing with our force-flags when
8752 * really needed and keeping the VMCS state as it is for most
8753 * VM-exits. */
8754 /* Update the guest interruptibility-state from the VMCS. */
8755 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8756
8757#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8758 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8759 AssertRC(rc);
8760#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8761 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8762 AssertRC(rc);
8763#endif
8764
8765 /*
8766 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8767 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8768 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8769 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8770 */
8771 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8772 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8773 {
8774 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8775 AssertRC(rc);
8776 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8777 }
8778 }
8779}
8780
8781
8782/**
8783 * Runs the guest code using VT-x the normal way.
8784 *
8785 * @returns VBox status code.
8786 * @param pVM Pointer to the VM.
8787 * @param pVCpu Pointer to the VMCPU.
8788 * @param pCtx Pointer to the guest-CPU context.
8789 *
8790 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8791 */
8792static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8793{
8794 VMXTRANSIENT VmxTransient;
8795 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8796 int rc = VERR_INTERNAL_ERROR_5;
8797 uint32_t cLoops = 0;
8798
8799 for (;; cLoops++)
8800 {
8801 Assert(!HMR0SuspendPending());
8802 HMVMX_ASSERT_CPU_SAFE();
8803
8804 /* Preparatory work for running guest code, this may force us to return
8805 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8806 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8807 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8808 if (rc != VINF_SUCCESS)
8809 break;
8810
8811 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8812 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8813 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8814
8815 /* Restore any residual host-state and save any bits shared between host
8816 and guest into the guest-CPU state. Re-enables interrupts! */
8817 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8818
8819 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8820 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8821 {
8822 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8823 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8824 return rc;
8825 }
8826
8827 /* Handle the VM-exit. */
8828 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8829 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8830 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8831 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8832 HMVMX_START_EXIT_DISPATCH_PROF();
8833#ifdef HMVMX_USE_FUNCTION_TABLE
8834 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8835#else
8836 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8837#endif
8838 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8839 if (rc != VINF_SUCCESS)
8840 break;
8841 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8842 {
8843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8844 rc = VINF_EM_RAW_INTERRUPT;
8845 break;
8846 }
8847 }
8848
8849 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8850 return rc;
8851}
8852
8853
8854/**
8855 * Single steps guest code using VT-x.
8856 *
8857 * @returns VBox status code.
8858 * @param pVM Pointer to the VM.
8859 * @param pVCpu Pointer to the VMCPU.
8860 * @param pCtx Pointer to the guest-CPU context.
8861 *
8862 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8863 */
8864static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8865{
8866 VMXTRANSIENT VmxTransient;
8867 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8868 int rc = VERR_INTERNAL_ERROR_5;
8869 uint32_t cLoops = 0;
8870 uint16_t uCsStart = pCtx->cs.Sel;
8871 uint64_t uRipStart = pCtx->rip;
8872
8873 for (;; cLoops++)
8874 {
8875 Assert(!HMR0SuspendPending());
8876 HMVMX_ASSERT_CPU_SAFE();
8877
8878 /* Preparatory work for running guest code, this may force us to return
8879 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8880 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8881 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8882 if (rc != VINF_SUCCESS)
8883 break;
8884
8885 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8886 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8887 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8888
8889 /* Restore any residual host-state and save any bits shared between host
8890 and guest into the guest-CPU state. Re-enables interrupts! */
8891 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8892
8893 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8894 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8895 {
8896 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8897 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8898 return rc;
8899 }
8900
8901 /* Handle the VM-exit. */
8902 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8903 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8904 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8905 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8906 HMVMX_START_EXIT_DISPATCH_PROF();
8907#ifdef HMVMX_USE_FUNCTION_TABLE
8908 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8909#else
8910 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8911#endif
8912 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8913 if (rc != VINF_SUCCESS)
8914 break;
8915 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8916 {
8917 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8918 rc = VINF_EM_RAW_INTERRUPT;
8919 break;
8920 }
8921
8922 /*
8923 * Did the RIP change, if so, consider it a single step.
8924 * Otherwise, make sure one of the TFs gets set.
8925 */
8926 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8927 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8928 AssertRCReturn(rc2, rc2);
8929 if ( pCtx->rip != uRipStart
8930 || pCtx->cs.Sel != uCsStart)
8931 {
8932 rc = VINF_EM_DBG_STEPPED;
8933 break;
8934 }
8935 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8936 }
8937
8938 /*
8939 * Clear the X86_EFL_TF if necessary.
8940 */
8941 if (pVCpu->hm.s.fClearTrapFlag)
8942 {
8943 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8944 AssertRCReturn(rc2, rc2);
8945 pVCpu->hm.s.fClearTrapFlag = false;
8946 pCtx->eflags.Bits.u1TF = 0;
8947 }
8948 /** @todo there seems to be issues with the resume flag when the monitor trap
8949 * flag is pending without being used. Seen early in bios init when
8950 * accessing APIC page in protected mode. */
8951
8952 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8953 return rc;
8954}
8955
8956
8957/**
8958 * Runs the guest code using VT-x.
8959 *
8960 * @returns VBox status code.
8961 * @param pVM Pointer to the VM.
8962 * @param pVCpu Pointer to the VMCPU.
8963 * @param pCtx Pointer to the guest-CPU context.
8964 */
8965VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8966{
8967 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8968 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8969 HMVMX_ASSERT_PREEMPT_SAFE();
8970
8971 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8972
8973 int rc;
8974 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8975 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8976 else
8977 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8978
8979 if (rc == VERR_EM_INTERPRETER)
8980 rc = VINF_EM_RAW_EMULATE_INSTR;
8981 else if (rc == VINF_EM_RESET)
8982 rc = VINF_EM_TRIPLE_FAULT;
8983
8984 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8985 if (RT_FAILURE(rc2))
8986 {
8987 pVCpu->hm.s.u32HMError = rc;
8988 rc = rc2;
8989 }
8990 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8991 return rc;
8992}
8993
8994
8995#ifndef HMVMX_USE_FUNCTION_TABLE
8996DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8997{
8998#ifdef DEBUG_ramshankar
8999# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9000# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9001#endif
9002 int rc;
9003 switch (rcReason)
9004 {
9005 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9006 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9007 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9008 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9009 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9010 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9011 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9012 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9013 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9014 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9015 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9016 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9017 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9018 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9019 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9020 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9021 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9022 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9023 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9024 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9025 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9026 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9027 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9028 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9029 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039
9040 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9041 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9042 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9043 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9044 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9045 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9046 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9047 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9048 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9049
9050 case VMX_EXIT_VMCLEAR:
9051 case VMX_EXIT_VMLAUNCH:
9052 case VMX_EXIT_VMPTRLD:
9053 case VMX_EXIT_VMPTRST:
9054 case VMX_EXIT_VMREAD:
9055 case VMX_EXIT_VMRESUME:
9056 case VMX_EXIT_VMWRITE:
9057 case VMX_EXIT_VMXOFF:
9058 case VMX_EXIT_VMXON:
9059 case VMX_EXIT_INVEPT:
9060 case VMX_EXIT_INVVPID:
9061 case VMX_EXIT_VMFUNC:
9062 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9063 break;
9064 default:
9065 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9066 break;
9067 }
9068 return rc;
9069}
9070#endif
9071
9072#ifdef DEBUG
9073/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9074# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9075 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9076
9077# define HMVMX_ASSERT_PREEMPT_CPUID() \
9078 do \
9079 { \
9080 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9081 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9082 } while (0)
9083
9084# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9085 do { \
9086 AssertPtr(pVCpu); \
9087 AssertPtr(pMixedCtx); \
9088 AssertPtr(pVmxTransient); \
9089 Assert(pVmxTransient->fVMEntryFailed == false); \
9090 Assert(ASMIntAreEnabled()); \
9091 HMVMX_ASSERT_PREEMPT_SAFE(); \
9092 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9093 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9094 HMVMX_ASSERT_PREEMPT_SAFE(); \
9095 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9096 HMVMX_ASSERT_PREEMPT_CPUID(); \
9097 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9098 } while (0)
9099
9100# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9101 do { \
9102 Log4Func(("\n")); \
9103 } while (0)
9104#else /* Release builds */
9105# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9106 do { \
9107 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9108 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9109 } while (0)
9110# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9111#endif
9112
9113
9114/**
9115 * Advances the guest RIP after reading it from the VMCS.
9116 *
9117 * @returns VBox status code.
9118 * @param pVCpu Pointer to the VMCPU.
9119 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9120 * out-of-sync. Make sure to update the required fields
9121 * before using them.
9122 * @param pVmxTransient Pointer to the VMX transient structure.
9123 *
9124 * @remarks No-long-jump zone!!!
9125 */
9126DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9127{
9128 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9129 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9130 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9131 AssertRCReturn(rc, rc);
9132
9133 pMixedCtx->rip += pVmxTransient->cbInstr;
9134 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9135
9136 /*
9137 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9138 * pending debug exception field as it takes care of priority of events.
9139 *
9140 * See Intel spec. 32.2.1 "Debug Exceptions".
9141 */
9142 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9143
9144 return rc;
9145}
9146
9147
9148/**
9149 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9150 * and update error record fields accordingly.
9151 *
9152 * @return VMX_IGS_* return codes.
9153 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9154 * wrong with the guest state.
9155 *
9156 * @param pVM Pointer to the VM.
9157 * @param pVCpu Pointer to the VMCPU.
9158 * @param pCtx Pointer to the guest-CPU state.
9159 *
9160 * @remarks This function assumes our cache of the VMCS controls
9161 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9162 */
9163static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9164{
9165#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9166#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9167 uError = (err); \
9168 break; \
9169 } else do { } while (0)
9170
9171 int rc;
9172 uint32_t uError = VMX_IGS_ERROR;
9173 uint32_t u32Val;
9174 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9175
9176 do
9177 {
9178 /*
9179 * CR0.
9180 */
9181 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9182 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9183 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9184 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9185 if (fUnrestrictedGuest)
9186 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9187
9188 uint32_t u32GuestCR0;
9189 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9190 AssertRCBreak(rc);
9191 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9192 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9193 if ( !fUnrestrictedGuest
9194 && (u32GuestCR0 & X86_CR0_PG)
9195 && !(u32GuestCR0 & X86_CR0_PE))
9196 {
9197 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9198 }
9199
9200 /*
9201 * CR4.
9202 */
9203 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9204 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9205
9206 uint32_t u32GuestCR4;
9207 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9208 AssertRCBreak(rc);
9209 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9210 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9211
9212 /*
9213 * IA32_DEBUGCTL MSR.
9214 */
9215 uint64_t u64Val;
9216 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9217 AssertRCBreak(rc);
9218 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9219 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9220 {
9221 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9222 }
9223 uint64_t u64DebugCtlMsr = u64Val;
9224
9225#ifdef VBOX_STRICT
9226 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9227 AssertRCBreak(rc);
9228 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9229#endif
9230 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9231
9232 /*
9233 * RIP and RFLAGS.
9234 */
9235 uint32_t u32Eflags;
9236#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9237 if (HMVMX_IS_64BIT_HOST_MODE())
9238 {
9239 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9240 AssertRCBreak(rc);
9241 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9242 if ( !fLongModeGuest
9243 || !pCtx->cs.Attr.n.u1Long)
9244 {
9245 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9246 }
9247 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9248 * must be identical if the "IA-32e mode guest" VM-entry
9249 * control is 1 and CS.L is 1. No check applies if the
9250 * CPU supports 64 linear-address bits. */
9251
9252 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9253 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9254 AssertRCBreak(rc);
9255 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9256 VMX_IGS_RFLAGS_RESERVED);
9257 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9258 u32Eflags = u64Val;
9259 }
9260 else
9261#endif
9262 {
9263 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9264 AssertRCBreak(rc);
9265 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9266 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9267 }
9268
9269 if ( fLongModeGuest
9270 || ( fUnrestrictedGuest
9271 && !(u32GuestCR0 & X86_CR0_PE)))
9272 {
9273 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9274 }
9275
9276 uint32_t u32EntryInfo;
9277 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9278 AssertRCBreak(rc);
9279 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9280 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9281 {
9282 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9283 }
9284
9285 /*
9286 * 64-bit checks.
9287 */
9288#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9289 if (HMVMX_IS_64BIT_HOST_MODE())
9290 {
9291 if ( fLongModeGuest
9292 && !fUnrestrictedGuest)
9293 {
9294 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9295 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9296 }
9297
9298 if ( !fLongModeGuest
9299 && (u32GuestCR4 & X86_CR4_PCIDE))
9300 {
9301 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9302 }
9303
9304 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9305 * 51:32 beyond the processor's physical-address width are 0. */
9306
9307 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9308 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9309 {
9310 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9311 }
9312
9313 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9314 AssertRCBreak(rc);
9315 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9316
9317 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9318 AssertRCBreak(rc);
9319 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9320 }
9321#endif
9322
9323 /*
9324 * PERF_GLOBAL MSR.
9325 */
9326 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9327 {
9328 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9329 AssertRCBreak(rc);
9330 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9331 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9332 }
9333
9334 /*
9335 * PAT MSR.
9336 */
9337 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9338 {
9339 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9340 AssertRCBreak(rc);
9341 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9342 for (unsigned i = 0; i < 8; i++)
9343 {
9344 uint8_t u8Val = (u64Val & 0x7);
9345 if ( u8Val != 0 /* UC */
9346 || u8Val != 1 /* WC */
9347 || u8Val != 4 /* WT */
9348 || u8Val != 5 /* WP */
9349 || u8Val != 6 /* WB */
9350 || u8Val != 7 /* UC- */)
9351 {
9352 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9353 }
9354 u64Val >>= 3;
9355 }
9356 }
9357
9358 /*
9359 * EFER MSR.
9360 */
9361 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9362 {
9363 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9364 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9365 AssertRCBreak(rc);
9366 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9367 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9368 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9369 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9370 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9371 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9372 VMX_IGS_EFER_LMA_PG_MISMATCH);
9373 }
9374
9375 /*
9376 * Segment registers.
9377 */
9378 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9379 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9380 if (!(u32Eflags & X86_EFL_VM))
9381 {
9382 /* CS */
9383 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9384 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9385 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9386 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9387 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9388 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9389 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9390 /* CS cannot be loaded with NULL in protected mode. */
9391 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9392 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9393 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9394 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9395 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9396 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9397 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9398 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9399 else
9400 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9401
9402 /* SS */
9403 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9404 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9405 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9406 if ( !(pCtx->cr0 & X86_CR0_PE)
9407 || pCtx->cs.Attr.n.u4Type == 3)
9408 {
9409 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9410 }
9411 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9412 {
9413 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9414 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9415 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9416 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9417 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9418 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9419 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9420 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9421 }
9422
9423 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9424 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9425 {
9426 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9427 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9428 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9429 || pCtx->ds.Attr.n.u4Type > 11
9430 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9431 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9432 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9433 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9434 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9435 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9436 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9437 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9438 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9439 }
9440 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9441 {
9442 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9443 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9444 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9445 || pCtx->es.Attr.n.u4Type > 11
9446 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9447 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9448 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9449 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9450 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9451 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9452 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9453 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9454 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9455 }
9456 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9457 {
9458 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9459 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9460 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9461 || pCtx->fs.Attr.n.u4Type > 11
9462 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9463 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9464 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9465 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9466 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9467 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9468 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9469 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9470 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9471 }
9472 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9473 {
9474 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9475 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9476 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9477 || pCtx->gs.Attr.n.u4Type > 11
9478 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9479 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9480 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9481 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9482 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9483 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9484 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9485 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9486 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9487 }
9488 /* 64-bit capable CPUs. */
9489#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9490 if (HMVMX_IS_64BIT_HOST_MODE())
9491 {
9492 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9493 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9494 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9495 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9496 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9497 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9498 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9499 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9500 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9501 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9502 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9503 }
9504#endif
9505 }
9506 else
9507 {
9508 /* V86 mode checks. */
9509 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9510 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9511 {
9512 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9513 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9514 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9515 }
9516 else
9517 {
9518 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9519 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9520 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9521 }
9522
9523 /* CS */
9524 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9525 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9526 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9527 /* SS */
9528 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9529 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9530 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9531 /* DS */
9532 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9533 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9534 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9535 /* ES */
9536 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9537 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9538 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9539 /* FS */
9540 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9541 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9542 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9543 /* GS */
9544 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9545 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9546 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9547 /* 64-bit capable CPUs. */
9548#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9549 if (HMVMX_IS_64BIT_HOST_MODE())
9550 {
9551 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9552 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9553 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9554 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9555 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9556 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9557 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9558 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9559 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9560 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9561 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9562 }
9563#endif
9564 }
9565
9566 /*
9567 * TR.
9568 */
9569 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9570 /* 64-bit capable CPUs. */
9571#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9572 if (HMVMX_IS_64BIT_HOST_MODE())
9573 {
9574 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9575 }
9576#endif
9577 if (fLongModeGuest)
9578 {
9579 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9580 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9581 }
9582 else
9583 {
9584 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9585 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9586 VMX_IGS_TR_ATTR_TYPE_INVALID);
9587 }
9588 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9589 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9590 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9591 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9592 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9593 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9594 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9595 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9596
9597 /*
9598 * GDTR and IDTR.
9599 */
9600#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9601 if (HMVMX_IS_64BIT_HOST_MODE())
9602 {
9603 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9604 AssertRCBreak(rc);
9605 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9606
9607 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9608 AssertRCBreak(rc);
9609 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9610 }
9611#endif
9612
9613 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9614 AssertRCBreak(rc);
9615 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9616
9617 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9618 AssertRCBreak(rc);
9619 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9620
9621 /*
9622 * Guest Non-Register State.
9623 */
9624 /* Activity State. */
9625 uint32_t u32ActivityState;
9626 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9627 AssertRCBreak(rc);
9628 HMVMX_CHECK_BREAK( !u32ActivityState
9629 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9630 VMX_IGS_ACTIVITY_STATE_INVALID);
9631 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9632 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9633 uint32_t u32IntrState;
9634 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9635 AssertRCBreak(rc);
9636 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9637 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9638 {
9639 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9640 }
9641
9642 /** @todo Activity state and injecting interrupts. Left as a todo since we
9643 * currently don't use activity states but ACTIVE. */
9644
9645 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9646 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9647
9648 /* Guest interruptibility-state. */
9649 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9650 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9651 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9652 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9653 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9654 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9655 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9656 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9657 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9658 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9659 {
9660 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9661 {
9662 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9663 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9664 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9665 }
9666 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9667 {
9668 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9669 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9670 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9671 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9672 }
9673 }
9674 /** @todo Assumes the processor is not in SMM. */
9675 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9676 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9677 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9678 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9679 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9680 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9681 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9682 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9683 {
9684 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9685 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9686 }
9687
9688 /* Pending debug exceptions. */
9689 if (HMVMX_IS_64BIT_HOST_MODE())
9690 {
9691 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9692 AssertRCBreak(rc);
9693 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9694 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9695 u32Val = u64Val; /* For pending debug exceptions checks below. */
9696 }
9697 else
9698 {
9699 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9700 AssertRCBreak(rc);
9701 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9702 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9703 }
9704
9705 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9706 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9707 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9708 {
9709 if ( (u32Eflags & X86_EFL_TF)
9710 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9711 {
9712 /* Bit 14 is PendingDebug.BS. */
9713 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9714 }
9715 if ( !(u32Eflags & X86_EFL_TF)
9716 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9717 {
9718 /* Bit 14 is PendingDebug.BS. */
9719 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9720 }
9721 }
9722
9723 /* VMCS link pointer. */
9724 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9725 AssertRCBreak(rc);
9726 if (u64Val != UINT64_C(0xffffffffffffffff))
9727 {
9728 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9729 /** @todo Bits beyond the processor's physical-address width MBZ. */
9730 /** @todo 32-bit located in memory referenced by value of this field (as a
9731 * physical address) must contain the processor's VMCS revision ID. */
9732 /** @todo SMM checks. */
9733 }
9734
9735 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9736 * not using Nested Paging? */
9737 if ( pVM->hm.s.fNestedPaging
9738 && !fLongModeGuest
9739 && CPUMIsGuestInPAEModeEx(pCtx))
9740 {
9741 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9742 AssertRCBreak(rc);
9743 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9744
9745 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9746 AssertRCBreak(rc);
9747 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9748
9749 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9750 AssertRCBreak(rc);
9751 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9752
9753 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9754 AssertRCBreak(rc);
9755 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9756 }
9757
9758 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9759 if (uError == VMX_IGS_ERROR)
9760 uError = VMX_IGS_REASON_NOT_FOUND;
9761 } while (0);
9762
9763 pVCpu->hm.s.u32HMError = uError;
9764 return uError;
9765
9766#undef HMVMX_ERROR_BREAK
9767#undef HMVMX_CHECK_BREAK
9768}
9769
9770/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9771/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9772/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9773
9774/** @name VM-exit handlers.
9775 * @{
9776 */
9777
9778/**
9779 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9780 */
9781HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9782{
9783 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9784 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9785 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9786 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9787 return VINF_SUCCESS;
9788 return VINF_EM_RAW_INTERRUPT;
9789}
9790
9791
9792/**
9793 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9794 */
9795HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9796{
9797 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9798 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9799
9800 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9801 AssertRCReturn(rc, rc);
9802
9803 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9804 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9805 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9806 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9807
9808 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9809 {
9810 /*
9811 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9812 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9813 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9814 *
9815 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9816 */
9817 VMXDispatchHostNmi();
9818 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9819 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9820 return VINF_SUCCESS;
9821 }
9822
9823 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9824 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9825 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9826 {
9827 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9828 return VINF_SUCCESS;
9829 }
9830 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9831 {
9832 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9833 return rc;
9834 }
9835
9836 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9837 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9838 switch (uIntType)
9839 {
9840 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9841 Assert(uVector == X86_XCPT_DB);
9842 /* no break */
9843 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9844 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9845 /* no break */
9846 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9847 {
9848 switch (uVector)
9849 {
9850 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9851 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9852 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9853 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9854 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9855 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9856#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9857 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9858 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9859 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9860 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9861 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9862 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9863 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9864 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9865 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9866 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9867 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9868 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9869#endif
9870 default:
9871 {
9872 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9873 AssertRCReturn(rc, rc);
9874
9875 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9876 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9877 {
9878 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9879 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9880 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9881
9882 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9883 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9884 AssertRCReturn(rc, rc);
9885 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9886 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9887 0 /* GCPtrFaultAddress */);
9888 AssertRCReturn(rc, rc);
9889 }
9890 else
9891 {
9892 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9893 pVCpu->hm.s.u32HMError = uVector;
9894 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9895 }
9896 break;
9897 }
9898 }
9899 break;
9900 }
9901
9902 default:
9903 {
9904 pVCpu->hm.s.u32HMError = uExitIntInfo;
9905 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9906 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9907 break;
9908 }
9909 }
9910 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9911 return rc;
9912}
9913
9914
9915/**
9916 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9917 */
9918HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9919{
9920 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9921
9922 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9923 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9924
9925 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9926 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9927 return VINF_SUCCESS;
9928}
9929
9930
9931/**
9932 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9933 */
9934HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9935{
9936 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9937 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
9938 {
9939 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9940 HMVMX_RETURN_UNEXPECTED_EXIT();
9941 }
9942
9943 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
9944
9945 /*
9946 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
9947 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
9948 */
9949 uint32_t uIntrState = 0;
9950 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9951 AssertRCReturn(rc, rc);
9952
9953 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
9954 if ( fBlockSti
9955 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
9956 {
9957 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9958 }
9959
9960 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
9961 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
9962
9963 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9964 return VINF_SUCCESS;
9965}
9966
9967
9968/**
9969 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9970 */
9971HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9972{
9973 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9974 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9975 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9976}
9977
9978
9979/**
9980 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9981 */
9982HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9983{
9984 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9985 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9986 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9987}
9988
9989
9990/**
9991 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9992 */
9993HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9994{
9995 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9996 PVM pVM = pVCpu->CTX_SUFF(pVM);
9997 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9998 if (RT_LIKELY(rc == VINF_SUCCESS))
9999 {
10000 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10001 Assert(pVmxTransient->cbInstr == 2);
10002 }
10003 else
10004 {
10005 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10006 rc = VERR_EM_INTERPRETER;
10007 }
10008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10009 return rc;
10010}
10011
10012
10013/**
10014 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10015 */
10016HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10017{
10018 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10019 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10020 AssertRCReturn(rc, rc);
10021
10022 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10023 return VINF_EM_RAW_EMULATE_INSTR;
10024
10025 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10026 HMVMX_RETURN_UNEXPECTED_EXIT();
10027}
10028
10029
10030/**
10031 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10032 */
10033HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10034{
10035 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10036 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10037 AssertRCReturn(rc, rc);
10038
10039 PVM pVM = pVCpu->CTX_SUFF(pVM);
10040 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10041 if (RT_LIKELY(rc == VINF_SUCCESS))
10042 {
10043 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10044 Assert(pVmxTransient->cbInstr == 2);
10045 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10046 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10047 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10048 }
10049 else
10050 {
10051 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
10052 rc = VERR_EM_INTERPRETER;
10053 }
10054 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10055 return rc;
10056}
10057
10058
10059/**
10060 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10061 */
10062HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10063{
10064 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10065 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10066 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10067 AssertRCReturn(rc, rc);
10068
10069 PVM pVM = pVCpu->CTX_SUFF(pVM);
10070 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10071 if (RT_LIKELY(rc == VINF_SUCCESS))
10072 {
10073 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10074 Assert(pVmxTransient->cbInstr == 3);
10075 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10076 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10077 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10078 }
10079 else
10080 {
10081 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10082 rc = VERR_EM_INTERPRETER;
10083 }
10084 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10085 return rc;
10086}
10087
10088
10089/**
10090 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10091 */
10092HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10093{
10094 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10095 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10096 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10097 AssertRCReturn(rc, rc);
10098
10099 PVM pVM = pVCpu->CTX_SUFF(pVM);
10100 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10101 if (RT_LIKELY(rc == VINF_SUCCESS))
10102 {
10103 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10104 Assert(pVmxTransient->cbInstr == 2);
10105 }
10106 else
10107 {
10108 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10109 rc = VERR_EM_INTERPRETER;
10110 }
10111 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10112 return rc;
10113}
10114
10115
10116/**
10117 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10118 */
10119HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10120{
10121 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10122
10123 int rc = VERR_NOT_SUPPORTED;
10124 if (GIMAreHypercallsEnabled(pVCpu))
10125 {
10126 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10127 AssertRCReturn(rc, rc);
10128
10129 rc = GIMHypercall(pVCpu, pMixedCtx);
10130 }
10131 if (rc != VINF_SUCCESS)
10132 {
10133 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10134 rc = VINF_SUCCESS;
10135 }
10136
10137 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10138 return rc;
10139}
10140
10141
10142/**
10143 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10144 */
10145HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10146{
10147 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10148 PVM pVM = pVCpu->CTX_SUFF(pVM);
10149 Assert(!pVM->hm.s.fNestedPaging);
10150
10151 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10152 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10153 AssertRCReturn(rc, rc);
10154
10155 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10156 rc = VBOXSTRICTRC_VAL(rc2);
10157 if (RT_LIKELY(rc == VINF_SUCCESS))
10158 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10159 else
10160 {
10161 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10162 pVmxTransient->uExitQualification, rc));
10163 }
10164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10165 return rc;
10166}
10167
10168
10169/**
10170 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10171 */
10172HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10173{
10174 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10175 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10176 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10177 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10178 AssertRCReturn(rc, rc);
10179
10180 PVM pVM = pVCpu->CTX_SUFF(pVM);
10181 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10182 if (RT_LIKELY(rc == VINF_SUCCESS))
10183 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10184 else
10185 {
10186 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10187 rc = VERR_EM_INTERPRETER;
10188 }
10189 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10190 return rc;
10191}
10192
10193
10194/**
10195 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10196 */
10197HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10198{
10199 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10200 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10201 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10202 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10203 AssertRCReturn(rc, rc);
10204
10205 PVM pVM = pVCpu->CTX_SUFF(pVM);
10206 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10207 rc = VBOXSTRICTRC_VAL(rc2);
10208 if (RT_LIKELY( rc == VINF_SUCCESS
10209 || rc == VINF_EM_HALT))
10210 {
10211 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10212 AssertRCReturn(rc3, rc3);
10213
10214 if ( rc == VINF_EM_HALT
10215 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10216 {
10217 rc = VINF_SUCCESS;
10218 }
10219 }
10220 else
10221 {
10222 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10223 rc = VERR_EM_INTERPRETER;
10224 }
10225 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10226 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10228 return rc;
10229}
10230
10231
10232/**
10233 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10234 */
10235HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10236{
10237 /*
10238 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10239 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10240 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10241 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10242 */
10243 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10244 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10245 HMVMX_RETURN_UNEXPECTED_EXIT();
10246}
10247
10248
10249/**
10250 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10251 */
10252HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10253{
10254 /*
10255 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10256 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10257 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10258 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10259 */
10260 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10261 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10262 HMVMX_RETURN_UNEXPECTED_EXIT();
10263}
10264
10265
10266/**
10267 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10268 */
10269HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10270{
10271 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10272 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10273 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10274 HMVMX_RETURN_UNEXPECTED_EXIT();
10275}
10276
10277
10278/**
10279 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10280 */
10281HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10282{
10283 /*
10284 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10285 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10286 * See Intel spec. 25.3 "Other Causes of VM-exits".
10287 */
10288 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10289 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10290 HMVMX_RETURN_UNEXPECTED_EXIT();
10291}
10292
10293
10294/**
10295 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10296 * VM-exit.
10297 */
10298HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10299{
10300 /*
10301 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10302 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10303 *
10304 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10305 * See Intel spec. "23.8 Restrictions on VMX operation".
10306 */
10307 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10308 return VINF_SUCCESS;
10309}
10310
10311
10312/**
10313 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10314 * VM-exit.
10315 */
10316HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10317{
10318 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10319 return VINF_EM_RESET;
10320}
10321
10322
10323/**
10324 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10325 */
10326HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10327{
10328 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10329 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10330 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10331 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10332 AssertRCReturn(rc, rc);
10333
10334 pMixedCtx->rip++;
10335 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10336 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10337 rc = VINF_SUCCESS;
10338 else
10339 rc = VINF_EM_HALT;
10340
10341 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10342 return rc;
10343}
10344
10345
10346/**
10347 * VM-exit handler for instructions that result in a #UD exception delivered to
10348 * the guest.
10349 */
10350HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10351{
10352 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10353 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10354 return VINF_SUCCESS;
10355}
10356
10357
10358/**
10359 * VM-exit handler for expiry of the VMX preemption timer.
10360 */
10361HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10362{
10363 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10364
10365 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10366 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10367
10368 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10369 PVM pVM = pVCpu->CTX_SUFF(pVM);
10370 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10371 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10372 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10373}
10374
10375
10376/**
10377 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10378 */
10379HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10380{
10381 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10382
10383 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10384 /** @todo check if XSETBV is supported by the recompiler. */
10385 return VERR_EM_INTERPRETER;
10386}
10387
10388
10389/**
10390 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10391 */
10392HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10393{
10394 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10395
10396 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10397 /** @todo implement EMInterpretInvpcid() */
10398 return VERR_EM_INTERPRETER;
10399}
10400
10401
10402/**
10403 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10404 * Error VM-exit.
10405 */
10406HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10407{
10408 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10409 AssertRCReturn(rc, rc);
10410
10411 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10412 AssertRCReturn(rc, rc);
10413
10414 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10415 NOREF(uInvalidReason);
10416
10417#ifdef VBOX_STRICT
10418 uint32_t uIntrState;
10419 HMVMXHCUINTREG uHCReg;
10420 uint64_t u64Val;
10421 uint32_t u32Val;
10422
10423 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10424 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10425 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10426 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10427 AssertRCReturn(rc, rc);
10428
10429 Log4(("uInvalidReason %u\n", uInvalidReason));
10430 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10431 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10432 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10433 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10434
10435 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10436 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10437 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10438 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10439 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10440 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10441 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10442 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10443 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10444 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10445 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10446 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10447#else
10448 NOREF(pVmxTransient);
10449#endif
10450
10451 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10452 return VERR_VMX_INVALID_GUEST_STATE;
10453}
10454
10455
10456/**
10457 * VM-exit handler for VM-entry failure due to an MSR-load
10458 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10459 */
10460HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10461{
10462 NOREF(pVmxTransient);
10463 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10464 HMVMX_RETURN_UNEXPECTED_EXIT();
10465}
10466
10467
10468/**
10469 * VM-exit handler for VM-entry failure due to a machine-check event
10470 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10471 */
10472HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10473{
10474 NOREF(pVmxTransient);
10475 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10476 HMVMX_RETURN_UNEXPECTED_EXIT();
10477}
10478
10479
10480/**
10481 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10482 * theory.
10483 */
10484HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10485{
10486 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10487 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10488 return VERR_VMX_UNDEFINED_EXIT_CODE;
10489}
10490
10491
10492/**
10493 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10494 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10495 * Conditional VM-exit.
10496 */
10497HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10498{
10499 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10500
10501 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10502 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10503 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10504 return VERR_EM_INTERPRETER;
10505 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10506 HMVMX_RETURN_UNEXPECTED_EXIT();
10507}
10508
10509
10510/**
10511 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10512 */
10513HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10514{
10515 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10516
10517 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10518 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10519 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10520 return VERR_EM_INTERPRETER;
10521 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10522 HMVMX_RETURN_UNEXPECTED_EXIT();
10523}
10524
10525
10526/**
10527 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10528 */
10529HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10530{
10531 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10532
10533 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10534 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10535 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10536 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10537 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10538 {
10539 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10540 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10541 }
10542 AssertRCReturn(rc, rc);
10543 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10544
10545#ifdef VBOX_STRICT
10546 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10547 {
10548 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10549 && pMixedCtx->ecx != MSR_K6_EFER)
10550 {
10551 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10552 HMVMX_RETURN_UNEXPECTED_EXIT();
10553 }
10554# if HC_ARCH_BITS == 64
10555 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10556 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10557 {
10558 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10559 HMVMX_RETURN_UNEXPECTED_EXIT();
10560 }
10561# endif
10562 }
10563#endif
10564
10565 PVM pVM = pVCpu->CTX_SUFF(pVM);
10566 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10567 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10568 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10569 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10570 if (RT_LIKELY(rc == VINF_SUCCESS))
10571 {
10572 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10573 Assert(pVmxTransient->cbInstr == 2);
10574 }
10575 return rc;
10576}
10577
10578
10579/**
10580 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10581 */
10582HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10583{
10584 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10585 PVM pVM = pVCpu->CTX_SUFF(pVM);
10586 int rc = VINF_SUCCESS;
10587
10588 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10589 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10590 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10591 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10592 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10593 {
10594 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10595 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10596 }
10597 AssertRCReturn(rc, rc);
10598 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10599
10600 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10601 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10602 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10603
10604 if (RT_LIKELY(rc == VINF_SUCCESS))
10605 {
10606 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10607
10608 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10609 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10610 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10611 {
10612 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10613 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10614 EMInterpretWrmsr() changes it. */
10615 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10616 }
10617 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10618 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10619 else if (pMixedCtx->ecx == MSR_K6_EFER)
10620 {
10621 /*
10622 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10623 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10624 * the other bits as well, SCE and NXE. See @bugref{7368}.
10625 */
10626 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10627 }
10628
10629 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10630 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10631 {
10632 switch (pMixedCtx->ecx)
10633 {
10634 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10635 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10636 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10637 case MSR_K8_FS_BASE: /* no break */
10638 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10639 case MSR_K6_EFER: /* already handled above */ break;
10640 default:
10641 {
10642 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10643 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10644#if HC_ARCH_BITS == 64
10645 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10646 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10647#endif
10648 break;
10649 }
10650 }
10651 }
10652#ifdef VBOX_STRICT
10653 else
10654 {
10655 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10656 switch (pMixedCtx->ecx)
10657 {
10658 case MSR_IA32_SYSENTER_CS:
10659 case MSR_IA32_SYSENTER_EIP:
10660 case MSR_IA32_SYSENTER_ESP:
10661 case MSR_K8_FS_BASE:
10662 case MSR_K8_GS_BASE:
10663 {
10664 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10665 HMVMX_RETURN_UNEXPECTED_EXIT();
10666 }
10667
10668 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10669 default:
10670 {
10671 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10672 {
10673 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10674 if (pMixedCtx->ecx != MSR_K6_EFER)
10675 {
10676 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10677 pMixedCtx->ecx));
10678 HMVMX_RETURN_UNEXPECTED_EXIT();
10679 }
10680 }
10681
10682#if HC_ARCH_BITS == 64
10683 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10684 {
10685 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10686 HMVMX_RETURN_UNEXPECTED_EXIT();
10687 }
10688#endif
10689 break;
10690 }
10691 }
10692 }
10693#endif /* VBOX_STRICT */
10694 }
10695 return rc;
10696}
10697
10698
10699/**
10700 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10701 */
10702HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10703{
10704 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10705
10706 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10707 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10708 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10709 return VERR_EM_INTERPRETER;
10710 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10711 HMVMX_RETURN_UNEXPECTED_EXIT();
10712}
10713
10714
10715/**
10716 * VM-exit handler for when the TPR value is lowered below the specified
10717 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10718 */
10719HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10720{
10721 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10722 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10723
10724 /*
10725 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10726 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10727 * resume guest execution.
10728 */
10729 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10730 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10731 return VINF_SUCCESS;
10732}
10733
10734
10735/**
10736 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10737 * VM-exit.
10738 *
10739 * @retval VINF_SUCCESS when guest execution can continue.
10740 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10741 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10742 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10743 * recompiler.
10744 */
10745HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10746{
10747 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10748 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10749 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10750 AssertRCReturn(rc, rc);
10751
10752 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10753 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10754 PVM pVM = pVCpu->CTX_SUFF(pVM);
10755 switch (uAccessType)
10756 {
10757 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10758 {
10759#if 0
10760 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10761 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10762#else
10763 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10764 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10765 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10766#endif
10767 AssertRCReturn(rc, rc);
10768
10769 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10770 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10771 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10772 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10773
10774 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10775 {
10776 case 0: /* CR0 */
10777 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10778 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10779 break;
10780 case 2: /* CR2 */
10781 /* Nothing to do here, CR2 it's not part of the VMCS. */
10782 break;
10783 case 3: /* CR3 */
10784 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10785 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10786 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10787 break;
10788 case 4: /* CR4 */
10789 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10790 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10791 break;
10792 case 8: /* CR8 */
10793 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10794 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10795 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10796 break;
10797 default:
10798 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10799 break;
10800 }
10801
10802 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10803 break;
10804 }
10805
10806 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10807 {
10808 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10809 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10810 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10811 AssertRCReturn(rc, rc);
10812 Assert( !pVM->hm.s.fNestedPaging
10813 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10814 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10815
10816 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10817 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10818 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10819
10820 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10821 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10822 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10823 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10824 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10825 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10826 break;
10827 }
10828
10829 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10830 {
10831 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10832 AssertRCReturn(rc, rc);
10833 rc = EMInterpretCLTS(pVM, pVCpu);
10834 AssertRCReturn(rc, rc);
10835 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10837 Log4(("CRX CLTS write rc=%d\n", rc));
10838 break;
10839 }
10840
10841 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10842 {
10843 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10844 AssertRCReturn(rc, rc);
10845 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10846 if (RT_LIKELY(rc == VINF_SUCCESS))
10847 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10848 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10849 Log4(("CRX LMSW write rc=%d\n", rc));
10850 break;
10851 }
10852
10853 default:
10854 {
10855 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10856 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10857 }
10858 }
10859
10860 /* Validate possible error codes. */
10861 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10862 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10863 if (RT_SUCCESS(rc))
10864 {
10865 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10866 AssertRCReturn(rc2, rc2);
10867 }
10868
10869 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10870 return rc;
10871}
10872
10873
10874/**
10875 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10876 * VM-exit.
10877 */
10878HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10879{
10880 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10881 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10882
10883 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10884 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10885 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10886 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10887 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10888 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10889 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10890 AssertRCReturn(rc2, rc2);
10891
10892 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10893 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10894 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10895 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10896 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10897 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10898 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10899 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10900
10901 /* I/O operation lookup arrays. */
10902 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10903 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10904
10905 VBOXSTRICTRC rcStrict;
10906 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10907 uint32_t const cbInstr = pVmxTransient->cbInstr;
10908 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10909 PVM pVM = pVCpu->CTX_SUFF(pVM);
10910 if (fIOString)
10911 {
10912#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10913 /*
10914 * INS/OUTS - I/O String instruction.
10915 *
10916 * Use instruction-information if available, otherwise fall back on
10917 * interpreting the instruction.
10918 */
10919 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10920 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10921 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10922 {
10923 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10924 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10925 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10926 AssertRCReturn(rc2, rc2);
10927 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10928 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10929 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10930 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10931 if (fIOWrite)
10932 {
10933 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10934 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10935 }
10936 else
10937 {
10938 /*
10939 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10940 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10941 * See Intel Instruction spec. for "INS".
10942 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10943 */
10944 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10945 }
10946 }
10947 else
10948 {
10949 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10950 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10951 AssertRCReturn(rc2, rc2);
10952 rcStrict = IEMExecOne(pVCpu);
10953 }
10954 /** @todo IEM needs to be setting these flags somehow. */
10955 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10956 fUpdateRipAlready = true;
10957#else
10958 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10959 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10960 if (RT_SUCCESS(rcStrict))
10961 {
10962 if (fIOWrite)
10963 {
10964 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10965 (DISCPUMODE)pDis->uAddrMode, cbValue);
10966 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10967 }
10968 else
10969 {
10970 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10971 (DISCPUMODE)pDis->uAddrMode, cbValue);
10972 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10973 }
10974 }
10975 else
10976 {
10977 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10978 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10979 }
10980#endif
10981 }
10982 else
10983 {
10984 /*
10985 * IN/OUT - I/O instruction.
10986 */
10987 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10988 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
10989 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10990 if (fIOWrite)
10991 {
10992 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10993 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10994 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10996 }
10997 else
10998 {
10999 uint32_t u32Result = 0;
11000 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11001 if (IOM_SUCCESS(rcStrict))
11002 {
11003 /* Save result of I/O IN instr. in AL/AX/EAX. */
11004 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11005 }
11006 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11007 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11009 }
11010 }
11011
11012 if (IOM_SUCCESS(rcStrict))
11013 {
11014 if (!fUpdateRipAlready)
11015 {
11016 pMixedCtx->rip += cbInstr;
11017 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11018 }
11019
11020 /*
11021 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11022 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11023 */
11024 if (fIOString)
11025 {
11026 /** @todo Single-step for INS/OUTS with REP prefix? */
11027 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11028 }
11029 else if (fStepping)
11030 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11031
11032 /*
11033 * If any I/O breakpoints are armed, we need to check if one triggered
11034 * and take appropriate action.
11035 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11036 */
11037 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11038 AssertRCReturn(rc2, rc2);
11039
11040 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11041 * execution engines about whether hyper BPs and such are pending. */
11042 uint32_t const uDr7 = pMixedCtx->dr[7];
11043 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11044 && X86_DR7_ANY_RW_IO(uDr7)
11045 && (pMixedCtx->cr4 & X86_CR4_DE))
11046 || DBGFBpIsHwIoArmed(pVM)))
11047 {
11048 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11049
11050 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11051 VMMRZCallRing3Disable(pVCpu);
11052 HM_DISABLE_PREEMPT_IF_NEEDED();
11053
11054 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
11055
11056 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11057 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11058 {
11059 /* Raise #DB. */
11060 if (fIsGuestDbgActive)
11061 ASMSetDR6(pMixedCtx->dr[6]);
11062 if (pMixedCtx->dr[7] != uDr7)
11063 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11064
11065 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11066 }
11067 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11068 else if ( rcStrict2 != VINF_SUCCESS
11069 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11070 rcStrict = rcStrict2;
11071
11072 HM_RESTORE_PREEMPT_IF_NEEDED();
11073 VMMRZCallRing3Enable(pVCpu);
11074 }
11075 }
11076
11077#ifdef DEBUG
11078 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11079 Assert(!fIOWrite);
11080 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11081 Assert(fIOWrite);
11082 else
11083 {
11084 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11085 * statuses, that the VMM device and some others may return. See
11086 * IOM_SUCCESS() for guidance. */
11087 AssertMsg( RT_FAILURE(rcStrict)
11088 || rcStrict == VINF_SUCCESS
11089 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11090 || rcStrict == VINF_EM_DBG_BREAKPOINT
11091 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11092 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11093 }
11094#endif
11095
11096 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11097 return VBOXSTRICTRC_TODO(rcStrict);
11098}
11099
11100
11101/**
11102 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11103 * VM-exit.
11104 */
11105HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11106{
11107 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11108
11109 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11110 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11111 AssertRCReturn(rc, rc);
11112 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11113 {
11114 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11115 AssertRCReturn(rc, rc);
11116 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11117 {
11118 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11119
11120 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11121 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11122
11123 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11124 Assert(!pVCpu->hm.s.Event.fPending);
11125 pVCpu->hm.s.Event.fPending = true;
11126 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11127 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11128 AssertRCReturn(rc, rc);
11129 if (fErrorCodeValid)
11130 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11131 else
11132 pVCpu->hm.s.Event.u32ErrCode = 0;
11133 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11134 && uVector == X86_XCPT_PF)
11135 {
11136 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11137 }
11138
11139 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11140 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11141 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11142 }
11143 }
11144
11145 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11146 * emulation. */
11147 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11148 return VERR_EM_INTERPRETER;
11149}
11150
11151
11152/**
11153 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11154 */
11155HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11156{
11157 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11158 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11159 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11160 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11161 AssertRCReturn(rc, rc);
11162 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11163 return VINF_EM_DBG_STEPPED;
11164}
11165
11166
11167/**
11168 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11169 */
11170HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11171{
11172 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11173
11174 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11175 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11176 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11177 return VINF_SUCCESS;
11178 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11179 return rc;
11180
11181#if 0
11182 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11183 * just sync the whole thing. */
11184 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11185#else
11186 /* Aggressive state sync. for now. */
11187 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11188 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11189 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11190#endif
11191 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11192 AssertRCReturn(rc, rc);
11193
11194 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11195 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11196 switch (uAccessType)
11197 {
11198 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11199 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11200 {
11201 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11202 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
11203 {
11204 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11205 }
11206
11207 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11208 GCPhys &= PAGE_BASE_GC_MASK;
11209 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11210 PVM pVM = pVCpu->CTX_SUFF(pVM);
11211 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11212 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11213
11214 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11215 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11216 CPUMCTX2CORE(pMixedCtx), GCPhys);
11217 rc = VBOXSTRICTRC_VAL(rc2);
11218 Log4(("ApicAccess rc=%d\n", rc));
11219 if ( rc == VINF_SUCCESS
11220 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11221 || rc == VERR_PAGE_NOT_PRESENT)
11222 {
11223 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11224 | HM_CHANGED_GUEST_RSP
11225 | HM_CHANGED_GUEST_RFLAGS
11226 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11227 rc = VINF_SUCCESS;
11228 }
11229 break;
11230 }
11231
11232 default:
11233 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11234 rc = VINF_EM_RAW_EMULATE_INSTR;
11235 break;
11236 }
11237
11238 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11239 return rc;
11240}
11241
11242
11243/**
11244 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11245 * VM-exit.
11246 */
11247HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11248{
11249 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11250
11251 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11252 if (pVmxTransient->fWasGuestDebugStateActive)
11253 {
11254 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11255 HMVMX_RETURN_UNEXPECTED_EXIT();
11256 }
11257
11258 int rc = VERR_INTERNAL_ERROR_5;
11259 if ( !DBGFIsStepping(pVCpu)
11260 && !pVCpu->hm.s.fSingleInstruction
11261 && !pVmxTransient->fWasHyperDebugStateActive)
11262 {
11263 /* Don't intercept MOV DRx and #DB any more. */
11264 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11265 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11266 AssertRCReturn(rc, rc);
11267
11268 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11269 {
11270#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11271 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11272 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11273 AssertRCReturn(rc, rc);
11274#endif
11275 }
11276
11277 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11278 VMMRZCallRing3Disable(pVCpu);
11279 HM_DISABLE_PREEMPT_IF_NEEDED();
11280
11281 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11282 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11283 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11284
11285 HM_RESTORE_PREEMPT_IF_NEEDED();
11286 VMMRZCallRing3Enable(pVCpu);
11287
11288#ifdef VBOX_WITH_STATISTICS
11289 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11290 AssertRCReturn(rc, rc);
11291 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11292 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11293 else
11294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11295#endif
11296 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11297 return VINF_SUCCESS;
11298 }
11299
11300 /*
11301 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11302 * Update the segment registers and DR7 from the CPU.
11303 */
11304 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11305 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11306 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11307 AssertRCReturn(rc, rc);
11308 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11309
11310 PVM pVM = pVCpu->CTX_SUFF(pVM);
11311 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11312 {
11313 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11314 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11315 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11316 if (RT_SUCCESS(rc))
11317 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11319 }
11320 else
11321 {
11322 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11323 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11324 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11325 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11326 }
11327
11328 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11329 if (RT_SUCCESS(rc))
11330 {
11331 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11332 AssertRCReturn(rc2, rc2);
11333 }
11334 return rc;
11335}
11336
11337
11338/**
11339 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11340 * Conditional VM-exit.
11341 */
11342HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11343{
11344 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11345 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11346
11347 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11348 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11349 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11350 return VINF_SUCCESS;
11351 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11352 return rc;
11353
11354 RTGCPHYS GCPhys = 0;
11355 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11356
11357#if 0
11358 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11359#else
11360 /* Aggressive state sync. for now. */
11361 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11362 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11363 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11364#endif
11365 AssertRCReturn(rc, rc);
11366
11367 /*
11368 * If we succeed, resume guest execution.
11369 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11370 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11371 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11372 * weird case. See @bugref{6043}.
11373 */
11374 PVM pVM = pVCpu->CTX_SUFF(pVM);
11375 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11376 rc = VBOXSTRICTRC_VAL(rc2);
11377 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11378 if ( rc == VINF_SUCCESS
11379 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11380 || rc == VERR_PAGE_NOT_PRESENT)
11381 {
11382 /* Successfully handled MMIO operation. */
11383 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11384 | HM_CHANGED_GUEST_RSP
11385 | HM_CHANGED_GUEST_RFLAGS
11386 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11387 rc = VINF_SUCCESS;
11388 }
11389 return rc;
11390}
11391
11392
11393/**
11394 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11395 * VM-exit.
11396 */
11397HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11398{
11399 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11400 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11401
11402 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11403 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11404 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11405 return VINF_SUCCESS;
11406 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11407 return rc;
11408
11409 RTGCPHYS GCPhys = 0;
11410 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11411 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11412#if 0
11413 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11414#else
11415 /* Aggressive state sync. for now. */
11416 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11417 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11418 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11419#endif
11420 AssertRCReturn(rc, rc);
11421
11422 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11423 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11424
11425 RTGCUINT uErrorCode = 0;
11426 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11427 uErrorCode |= X86_TRAP_PF_ID;
11428 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11429 uErrorCode |= X86_TRAP_PF_RW;
11430 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11431 uErrorCode |= X86_TRAP_PF_P;
11432
11433 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11434
11435 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11436 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11437
11438 /* Handle the pagefault trap for the nested shadow table. */
11439 PVM pVM = pVCpu->CTX_SUFF(pVM);
11440 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11441 TRPMResetTrap(pVCpu);
11442
11443 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11444 if ( rc == VINF_SUCCESS
11445 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11446 || rc == VERR_PAGE_NOT_PRESENT)
11447 {
11448 /* Successfully synced our nested page tables. */
11449 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11450 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11451 | HM_CHANGED_GUEST_RSP
11452 | HM_CHANGED_GUEST_RFLAGS);
11453 return VINF_SUCCESS;
11454 }
11455
11456 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11457 return rc;
11458}
11459
11460/** @} */
11461
11462/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11463/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11464/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11465
11466/** @name VM-exit exception handlers.
11467 * @{
11468 */
11469
11470/**
11471 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11472 */
11473static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11474{
11475 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11476 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11477
11478 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11479 AssertRCReturn(rc, rc);
11480
11481 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11482 {
11483 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11484 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11485
11486 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11487 * provides VM-exit instruction length. If this causes problem later,
11488 * disassemble the instruction like it's done on AMD-V. */
11489 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11490 AssertRCReturn(rc2, rc2);
11491 return rc;
11492 }
11493
11494 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11495 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11496 return rc;
11497}
11498
11499
11500/**
11501 * VM-exit exception handler for #BP (Breakpoint exception).
11502 */
11503static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11504{
11505 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11507
11508 /** @todo Try optimize this by not saving the entire guest state unless
11509 * really needed. */
11510 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11511 AssertRCReturn(rc, rc);
11512
11513 PVM pVM = pVCpu->CTX_SUFF(pVM);
11514 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11515 if (rc == VINF_EM_RAW_GUEST_TRAP)
11516 {
11517 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11518 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11519 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11520 AssertRCReturn(rc, rc);
11521
11522 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11523 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11524 }
11525
11526 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11527 return rc;
11528}
11529
11530
11531/**
11532 * VM-exit exception handler for #DB (Debug exception).
11533 */
11534static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11535{
11536 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11537 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11538 Log6(("XcptDB\n"));
11539
11540 /*
11541 * Get the DR6-like values from the exit qualification and pass it to DBGF
11542 * for processing.
11543 */
11544 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11545 AssertRCReturn(rc, rc);
11546
11547 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11548 uint64_t uDR6 = X86_DR6_INIT_VAL;
11549 uDR6 |= ( pVmxTransient->uExitQualification
11550 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11551
11552 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11553 if (rc == VINF_EM_RAW_GUEST_TRAP)
11554 {
11555 /*
11556 * The exception was for the guest. Update DR6, DR7.GD and
11557 * IA32_DEBUGCTL.LBR before forwarding it.
11558 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11559 */
11560 VMMRZCallRing3Disable(pVCpu);
11561 HM_DISABLE_PREEMPT_IF_NEEDED();
11562
11563 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11564 pMixedCtx->dr[6] |= uDR6;
11565 if (CPUMIsGuestDebugStateActive(pVCpu))
11566 ASMSetDR6(pMixedCtx->dr[6]);
11567
11568 HM_RESTORE_PREEMPT_IF_NEEDED();
11569 VMMRZCallRing3Enable(pVCpu);
11570
11571 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11572 AssertRCReturn(rc, rc);
11573
11574 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11575 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11576
11577 /* Paranoia. */
11578 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11579 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11580
11581 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11582 AssertRCReturn(rc, rc);
11583
11584 /*
11585 * Raise #DB in the guest.
11586 *
11587 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11588 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11589 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11590 *
11591 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11592 */
11593 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11594 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11595 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11596 AssertRCReturn(rc, rc);
11597 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11598 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11599 return VINF_SUCCESS;
11600 }
11601
11602 /*
11603 * Not a guest trap, must be a hypervisor related debug event then.
11604 * Update DR6 in case someone is interested in it.
11605 */
11606 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11607 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11608 CPUMSetHyperDR6(pVCpu, uDR6);
11609
11610 return rc;
11611}
11612
11613
11614/**
11615 * VM-exit exception handler for #NM (Device-not-available exception: floating
11616 * point exception).
11617 */
11618static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11619{
11620 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11621
11622 /* We require CR0 and EFER. EFER is always up-to-date. */
11623 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11624 AssertRCReturn(rc, rc);
11625
11626 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11627 VMMRZCallRing3Disable(pVCpu);
11628 HM_DISABLE_PREEMPT_IF_NEEDED();
11629
11630 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11631 if (pVmxTransient->fWasGuestFPUStateActive)
11632 {
11633 rc = VINF_EM_RAW_GUEST_TRAP;
11634 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11635 }
11636 else
11637 {
11638#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11639 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11640#endif
11641 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11642 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11643 }
11644
11645 HM_RESTORE_PREEMPT_IF_NEEDED();
11646 VMMRZCallRing3Enable(pVCpu);
11647
11648 if (rc == VINF_SUCCESS)
11649 {
11650 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11651 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11652 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11653 pVCpu->hm.s.fUseGuestFpu = true;
11654 }
11655 else
11656 {
11657 /* Forward #NM to the guest. */
11658 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11659 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11660 AssertRCReturn(rc, rc);
11661 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11662 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11663 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11664 }
11665
11666 return VINF_SUCCESS;
11667}
11668
11669
11670/**
11671 * VM-exit exception handler for #GP (General-protection exception).
11672 *
11673 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11674 */
11675static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11676{
11677 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11678 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11679
11680 int rc = VERR_INTERNAL_ERROR_5;
11681 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11682 {
11683#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11684 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11685 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11686 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11687 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11688 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11689 AssertRCReturn(rc, rc);
11690 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11691 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11692 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11693 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11694 return rc;
11695#else
11696 /* We don't intercept #GP. */
11697 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11698 NOREF(pVmxTransient);
11699 return VERR_VMX_UNEXPECTED_EXCEPTION;
11700#endif
11701 }
11702
11703 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11704 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11705
11706 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11707 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11708 AssertRCReturn(rc, rc);
11709
11710 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11711 uint32_t cbOp = 0;
11712 PVM pVM = pVCpu->CTX_SUFF(pVM);
11713 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11714 if (RT_SUCCESS(rc))
11715 {
11716 rc = VINF_SUCCESS;
11717 Assert(cbOp == pDis->cbInstr);
11718 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11719 switch (pDis->pCurInstr->uOpcode)
11720 {
11721 case OP_CLI:
11722 {
11723 pMixedCtx->eflags.Bits.u1IF = 0;
11724 pMixedCtx->rip += pDis->cbInstr;
11725 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11726 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11727 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11728 break;
11729 }
11730
11731 case OP_STI:
11732 {
11733 pMixedCtx->eflags.Bits.u1IF = 1;
11734 pMixedCtx->rip += pDis->cbInstr;
11735 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11736 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11737 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11738 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11739 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11740 break;
11741 }
11742
11743 case OP_HLT:
11744 {
11745 rc = VINF_EM_HALT;
11746 pMixedCtx->rip += pDis->cbInstr;
11747 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11748 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11749 break;
11750 }
11751
11752 case OP_POPF:
11753 {
11754 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11755 uint32_t cbParm;
11756 uint32_t uMask;
11757 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11758 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11759 {
11760 cbParm = 4;
11761 uMask = 0xffffffff;
11762 }
11763 else
11764 {
11765 cbParm = 2;
11766 uMask = 0xffff;
11767 }
11768
11769 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11770 RTGCPTR GCPtrStack = 0;
11771 X86EFLAGS Eflags;
11772 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11773 &GCPtrStack);
11774 if (RT_SUCCESS(rc))
11775 {
11776 Assert(sizeof(Eflags.u32) >= cbParm);
11777 Eflags.u32 = 0;
11778 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11779 }
11780 if (RT_FAILURE(rc))
11781 {
11782 rc = VERR_EM_INTERPRETER;
11783 break;
11784 }
11785 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11786 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11787 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11788 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11789 pMixedCtx->esp += cbParm;
11790 pMixedCtx->esp &= uMask;
11791 pMixedCtx->rip += pDis->cbInstr;
11792 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11793 | HM_CHANGED_GUEST_RSP
11794 | HM_CHANGED_GUEST_RFLAGS);
11795 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11796 if (fStepping)
11797 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11798
11799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11800 break;
11801 }
11802
11803 case OP_PUSHF:
11804 {
11805 uint32_t cbParm;
11806 uint32_t uMask;
11807 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11808 {
11809 cbParm = 4;
11810 uMask = 0xffffffff;
11811 }
11812 else
11813 {
11814 cbParm = 2;
11815 uMask = 0xffff;
11816 }
11817
11818 /* Get the stack pointer & push the contents of eflags onto the stack. */
11819 RTGCPTR GCPtrStack = 0;
11820 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11821 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11822 if (RT_FAILURE(rc))
11823 {
11824 rc = VERR_EM_INTERPRETER;
11825 break;
11826 }
11827 X86EFLAGS Eflags = pMixedCtx->eflags;
11828 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11829 Eflags.Bits.u1RF = 0;
11830 Eflags.Bits.u1VM = 0;
11831
11832 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11833 if (RT_FAILURE(rc))
11834 {
11835 rc = VERR_EM_INTERPRETER;
11836 break;
11837 }
11838 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11839 pMixedCtx->esp -= cbParm;
11840 pMixedCtx->esp &= uMask;
11841 pMixedCtx->rip += pDis->cbInstr;
11842 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11843 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11844 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11845 break;
11846 }
11847
11848 case OP_IRET:
11849 {
11850 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11851 * instruction reference. */
11852 RTGCPTR GCPtrStack = 0;
11853 uint32_t uMask = 0xffff;
11854 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11855 uint16_t aIretFrame[3];
11856 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11857 {
11858 rc = VERR_EM_INTERPRETER;
11859 break;
11860 }
11861 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11862 &GCPtrStack);
11863 if (RT_SUCCESS(rc))
11864 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11865 if (RT_FAILURE(rc))
11866 {
11867 rc = VERR_EM_INTERPRETER;
11868 break;
11869 }
11870 pMixedCtx->eip = 0;
11871 pMixedCtx->ip = aIretFrame[0];
11872 pMixedCtx->cs.Sel = aIretFrame[1];
11873 pMixedCtx->cs.ValidSel = aIretFrame[1];
11874 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11875 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11876 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11877 pMixedCtx->sp += sizeof(aIretFrame);
11878 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11879 | HM_CHANGED_GUEST_SEGMENT_REGS
11880 | HM_CHANGED_GUEST_RSP
11881 | HM_CHANGED_GUEST_RFLAGS);
11882 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11883 if (fStepping)
11884 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11885 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11886 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11887 break;
11888 }
11889
11890 case OP_INT:
11891 {
11892 uint16_t uVector = pDis->Param1.uValue & 0xff;
11893 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11894 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11895 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11896 break;
11897 }
11898
11899 case OP_INTO:
11900 {
11901 if (pMixedCtx->eflags.Bits.u1OF)
11902 {
11903 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11904 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11905 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11906 }
11907 break;
11908 }
11909
11910 default:
11911 {
11912 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11913 EMCODETYPE_SUPERVISOR);
11914 rc = VBOXSTRICTRC_VAL(rc2);
11915 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11916 /** @todo We have to set pending-debug exceptions here when the guest is
11917 * single-stepping depending on the instruction that was interpreted. */
11918 Log4(("#GP rc=%Rrc\n", rc));
11919 break;
11920 }
11921 }
11922 }
11923 else
11924 rc = VERR_EM_INTERPRETER;
11925
11926 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11927 ("#GP Unexpected rc=%Rrc\n", rc));
11928 return rc;
11929}
11930
11931
11932#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11933/**
11934 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11935 * the exception reported in the VMX transient structure back into the VM.
11936 *
11937 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11938 * up-to-date.
11939 */
11940static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11941{
11942 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11943
11944 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11945 hmR0VmxCheckExitDueToEventDelivery(). */
11946 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11947 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11948 AssertRCReturn(rc, rc);
11949 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11950
11951#ifdef DEBUG_ramshankar
11952 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11953 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11954 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11955#endif
11956
11957 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11958 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11959 return VINF_SUCCESS;
11960}
11961#endif
11962
11963
11964/**
11965 * VM-exit exception handler for #PF (Page-fault exception).
11966 */
11967static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11968{
11969 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11970 PVM pVM = pVCpu->CTX_SUFF(pVM);
11971 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11972 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11973 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11974 AssertRCReturn(rc, rc);
11975
11976#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11977 if (pVM->hm.s.fNestedPaging)
11978 {
11979 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11980 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11981 {
11982 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11983 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11984 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11985 }
11986 else
11987 {
11988 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11989 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11990 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11991 }
11992 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11993 return rc;
11994 }
11995#else
11996 Assert(!pVM->hm.s.fNestedPaging);
11997 NOREF(pVM);
11998#endif
11999
12000 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12001 AssertRCReturn(rc, rc);
12002
12003 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12004 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12005
12006 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12007 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12008 (RTGCPTR)pVmxTransient->uExitQualification);
12009
12010 Log4(("#PF: rc=%Rrc\n", rc));
12011 if (rc == VINF_SUCCESS)
12012 {
12013 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12014 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12015 * memory? We don't update the whole state here... */
12016 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12017 | HM_CHANGED_GUEST_RSP
12018 | HM_CHANGED_GUEST_RFLAGS
12019 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12020 TRPMResetTrap(pVCpu);
12021 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12022 return rc;
12023 }
12024 else if (rc == VINF_EM_RAW_GUEST_TRAP)
12025 {
12026 if (!pVmxTransient->fVectoringPF)
12027 {
12028 /* It's a guest page fault and needs to be reflected to the guest. */
12029 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12030 TRPMResetTrap(pVCpu);
12031 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12032 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12033 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12034 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12035 }
12036 else
12037 {
12038 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12039 TRPMResetTrap(pVCpu);
12040 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12041 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12042 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12043 }
12044
12045 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12046 return VINF_SUCCESS;
12047 }
12048
12049 TRPMResetTrap(pVCpu);
12050 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12051 return rc;
12052}
12053
12054/** @} */
12055
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette