VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 52166

Last change on this file since 52166 was 52166, checked in by vboxsync, 10 years ago

VMM/HMVMXR0: Move comment to the right place.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 499.3 KB
Line 
1/* $Id: HMVMXR0.cpp 52166 2014-07-24 12:04:15Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
41# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_CHECK_GUEST_STATE
44# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
45# define HMVMX_ALWAYS_TRAP_PF
46# define HMVMX_ALWAYS_SWAP_FPU_STATE
47# define HMVMX_ALWAYS_FLUSH_TLB
48# define HMVMX_ALWAYS_SWAP_EFER
49#endif
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define HMVMX_IS_64BIT_HOST_MODE() (true)
57typedef RTHCUINTREG HMVMXHCUINTREG;
58#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
59extern "C" uint32_t g_fVMXIs64bitHost;
60# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
61typedef uint64_t HMVMXHCUINTREG;
62#else
63# define HMVMX_IS_64BIT_HOST_MODE() (false)
64typedef RTHCUINTREG HMVMXHCUINTREG;
65#endif
66
67/** Use the function table. */
68#define HMVMX_USE_FUNCTION_TABLE
69
70/** Determine which tagged-TLB flush handler to use. */
71#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
72#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
73#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
74#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
75
76/** @name Updated-guest-state flags.
77 * @{ */
78#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
79#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
80#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
81#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
82#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
83#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
84#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
85#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
86#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
87#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
88#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
89#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
90#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
91#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
92#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
93#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
94#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
95#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
96#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
97#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
98#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
99 | HMVMX_UPDATED_GUEST_RSP \
100 | HMVMX_UPDATED_GUEST_RFLAGS \
101 | HMVMX_UPDATED_GUEST_CR0 \
102 | HMVMX_UPDATED_GUEST_CR3 \
103 | HMVMX_UPDATED_GUEST_CR4 \
104 | HMVMX_UPDATED_GUEST_GDTR \
105 | HMVMX_UPDATED_GUEST_IDTR \
106 | HMVMX_UPDATED_GUEST_LDTR \
107 | HMVMX_UPDATED_GUEST_TR \
108 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
109 | HMVMX_UPDATED_GUEST_DEBUG \
110 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
112 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
113 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
114 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
115 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
116 | HMVMX_UPDATED_GUEST_INTR_STATE \
117 | HMVMX_UPDATED_GUEST_APIC_STATE)
118/** @} */
119
120/** @name
121 * Flags to skip redundant reads of some common VMCS fields that are not part of
122 * the guest-CPU state but are in the transient structure.
123 */
124#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
125#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
131/** @} */
132
133/** @name
134 * States of the VMCS.
135 *
136 * This does not reflect all possible VMCS states but currently only those
137 * needed for maintaining the VMCS consistently even when thread-context hooks
138 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
139 */
140#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
141#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
142#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
143/** @} */
144
145/**
146 * Exception bitmap mask for real-mode guests (real-on-v86).
147 *
148 * We need to intercept all exceptions manually (except #PF). #NM is also
149 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
150 * even in real-mode if we have Nested Paging support.
151 */
152#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
153 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
154 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
155 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
156 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
157 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
158 | RT_BIT(X86_XCPT_XF))
159
160/**
161 * Exception bitmap mask for all contributory exceptions.
162 *
163 * Page fault is deliberately excluded here as it's conditional as to whether
164 * it's contributory or benign. Page faults are handled separately.
165 */
166#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
167 | RT_BIT(X86_XCPT_DE))
168
169/** Maximum VM-instruction error number. */
170#define HMVMX_INSTR_ERROR_MAX 28
171
172/** Profiling macro. */
173#ifdef HM_PROFILE_EXIT_DISPATCH
174# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
176#else
177# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
178# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
179#endif
180
181/** Assert that preemption is disabled or covered by thread-context hooks. */
182#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
184
185/** Assert that we haven't migrated CPUs when thread-context hooks are not
186 * used. */
187#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
188 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
189 ("Illegal migration! Entered on CPU %u Current %u\n", \
190 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
191
192/** Helper macro for VM-exit handlers called unexpectedly. */
193#define HMVMX_RETURN_UNEXPECTED_EXIT() \
194 do { \
195 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
196 return VERR_VMX_UNEXPECTED_EXIT; \
197 } while (0)
198
199
200/*******************************************************************************
201* Structures and Typedefs *
202*******************************************************************************/
203/**
204 * VMX transient state.
205 *
206 * A state structure for holding miscellaneous information across
207 * VMX non-root operation and restored after the transition.
208 */
209typedef struct VMXTRANSIENT
210{
211 /** The host's rflags/eflags. */
212 RTCCUINTREG uEflags;
213#if HC_ARCH_BITS == 32
214 uint32_t u32Alignment0;
215#endif
216 /** The guest's TPR value used for TPR shadowing. */
217 uint8_t u8GuestTpr;
218 /** Alignment. */
219 uint8_t abAlignment0[7];
220
221 /** The basic VM-exit reason. */
222 uint16_t uExitReason;
223 /** Alignment. */
224 uint16_t u16Alignment0;
225 /** The VM-exit interruption error code. */
226 uint32_t uExitIntErrorCode;
227 /** The VM-exit exit qualification. */
228 uint64_t uExitQualification;
229
230 /** The VM-exit interruption-information field. */
231 uint32_t uExitIntInfo;
232 /** The VM-exit instruction-length field. */
233 uint32_t cbInstr;
234 /** The VM-exit instruction-information field. */
235 union
236 {
237 /** Plain unsigned int representation. */
238 uint32_t u;
239 /** INS and OUTS information. */
240 struct
241 {
242 uint32_t u6Reserved0 : 7;
243 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
244 uint32_t u3AddrSize : 3;
245 uint32_t u5Reserved1 : 5;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 uint32_t uReserved2 : 14;
249 } StrIo;
250 } ExitInstrInfo;
251 /** Whether the VM-entry failed or not. */
252 bool fVMEntryFailed;
253 /** Alignment. */
254 uint8_t abAlignment1[3];
255
256 /** The VM-entry interruption-information field. */
257 uint32_t uEntryIntInfo;
258 /** The VM-entry exception error code field. */
259 uint32_t uEntryXcptErrorCode;
260 /** The VM-entry instruction length field. */
261 uint32_t cbEntryInstr;
262
263 /** IDT-vectoring information field. */
264 uint32_t uIdtVectoringInfo;
265 /** IDT-vectoring error code. */
266 uint32_t uIdtVectoringErrorCode;
267
268 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
269 uint32_t fVmcsFieldsRead;
270
271 /** Whether the guest FPU was active at the time of VM-exit. */
272 bool fWasGuestFPUStateActive;
273 /** Whether the guest debug state was active at the time of VM-exit. */
274 bool fWasGuestDebugStateActive;
275 /** Whether the hyper debug state was active at the time of VM-exit. */
276 bool fWasHyperDebugStateActive;
277 /** Whether TSC-offsetting should be setup before VM-entry. */
278 bool fUpdateTscOffsettingAndPreemptTimer;
279 /** Whether the VM-exit was caused by a page-fault during delivery of a
280 * contributory exception or a page-fault. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns VBox status code.
323 * @param pVCpu Pointer to the VMCPU.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337
338/*******************************************************************************
339* Internal Functions *
340*******************************************************************************/
341static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
342static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
343static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
344 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
345#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
346static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
347#endif
348#ifndef HMVMX_USE_FUNCTION_TABLE
349DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
350# define HMVMX_EXIT_DECL static int
351#else
352# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
353#endif
354
355/** @name VM-exit handlers.
356 * @{
357 */
358static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
359static FNVMXEXITHANDLER hmR0VmxExitExtInt;
360static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
361static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
362static FNVMXEXITHANDLER hmR0VmxExitSipi;
363static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
364static FNVMXEXITHANDLER hmR0VmxExitSmi;
365static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
366static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
367static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
368static FNVMXEXITHANDLER hmR0VmxExitCpuid;
369static FNVMXEXITHANDLER hmR0VmxExitGetsec;
370static FNVMXEXITHANDLER hmR0VmxExitHlt;
371static FNVMXEXITHANDLER hmR0VmxExitInvd;
372static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
373static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
374static FNVMXEXITHANDLER hmR0VmxExitVmcall;
375static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
376static FNVMXEXITHANDLER hmR0VmxExitRsm;
377static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
378static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
379static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
380static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
381static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
382static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
383static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
384static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
385static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
386static FNVMXEXITHANDLER hmR0VmxExitMwait;
387static FNVMXEXITHANDLER hmR0VmxExitMtf;
388static FNVMXEXITHANDLER hmR0VmxExitMonitor;
389static FNVMXEXITHANDLER hmR0VmxExitPause;
390static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
391static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
392static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
393static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
394static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
395static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
396static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
397static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
398static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
399static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
400static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
401static FNVMXEXITHANDLER hmR0VmxExitRdrand;
402static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
403/** @} */
404
405static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
412static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413#endif
414static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
415
416/*******************************************************************************
417* Global Variables *
418*******************************************************************************/
419#ifdef HMVMX_USE_FUNCTION_TABLE
420
421/**
422 * VMX_EXIT dispatch table.
423 */
424static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
425{
426 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
427 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
428 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
429 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
430 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
431 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
432 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
433 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
434 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
435 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
436 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
437 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
438 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
439 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
440 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
441 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
442 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
443 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
444 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
445 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
446 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
447 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
448 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
449 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
450 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
451 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
452 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
453 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
454 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
455 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
456 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
457 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
458 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
459 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
460 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
461 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
462 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
463 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
464 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
465 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
466 /* 40 UNDEFINED */ hmR0VmxExitPause,
467 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
468 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
469 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
470 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
471 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
472 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
473 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
474 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
475 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
476 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
477 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
478 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
479 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
480 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
481 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
482 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
483 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
484 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
485 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
486};
487#endif /* HMVMX_USE_FUNCTION_TABLE */
488
489#ifdef VBOX_STRICT
490static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
491{
492 /* 0 */ "(Not Used)",
493 /* 1 */ "VMCALL executed in VMX root operation.",
494 /* 2 */ "VMCLEAR with invalid physical address.",
495 /* 3 */ "VMCLEAR with VMXON pointer.",
496 /* 4 */ "VMLAUNCH with non-clear VMCS.",
497 /* 5 */ "VMRESUME with non-launched VMCS.",
498 /* 6 */ "VMRESUME after VMXOFF",
499 /* 7 */ "VM entry with invalid control fields.",
500 /* 8 */ "VM entry with invalid host state fields.",
501 /* 9 */ "VMPTRLD with invalid physical address.",
502 /* 10 */ "VMPTRLD with VMXON pointer.",
503 /* 11 */ "VMPTRLD with incorrect revision identifier.",
504 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
505 /* 13 */ "VMWRITE to read-only VMCS component.",
506 /* 14 */ "(Not Used)",
507 /* 15 */ "VMXON executed in VMX root operation.",
508 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
509 /* 17 */ "VM entry with non-launched executing VMCS.",
510 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
511 /* 19 */ "VMCALL with non-clear VMCS.",
512 /* 20 */ "VMCALL with invalid VM-exit control fields.",
513 /* 21 */ "(Not Used)",
514 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
515 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
516 /* 24 */ "VMCALL with invalid SMM-monitor features.",
517 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
518 /* 26 */ "VM entry with events blocked by MOV SS.",
519 /* 27 */ "(Not Used)",
520 /* 28 */ "Invalid operand to INVEPT/INVVPID."
521};
522#endif /* VBOX_STRICT */
523
524
525
526/**
527 * Updates the VM's last error record. If there was a VMX instruction error,
528 * reads the error data from the VMCS and updates VCPU's last error record as
529 * well.
530 *
531 * @param pVM Pointer to the VM.
532 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
533 * VERR_VMX_UNABLE_TO_START_VM or
534 * VERR_VMX_INVALID_VMCS_FIELD).
535 * @param rc The error code.
536 */
537static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
538{
539 AssertPtr(pVM);
540 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
541 || rc == VERR_VMX_UNABLE_TO_START_VM)
542 {
543 AssertPtrReturnVoid(pVCpu);
544 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
545 }
546 pVM->hm.s.lLastError = rc;
547}
548
549
550/**
551 * Reads the VM-entry interruption-information field from the VMCS into the VMX
552 * transient structure.
553 *
554 * @returns VBox status code.
555 * @param pVmxTransient Pointer to the VMX transient structure.
556 *
557 * @remarks No-long-jump zone!!!
558 */
559DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
560{
561 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
562 AssertRCReturn(rc, rc);
563 return VINF_SUCCESS;
564}
565
566
567/**
568 * Reads the VM-entry exception error code field from the VMCS into
569 * the VMX transient structure.
570 *
571 * @returns VBox status code.
572 * @param pVmxTransient Pointer to the VMX transient structure.
573 *
574 * @remarks No-long-jump zone!!!
575 */
576DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
577{
578 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
579 AssertRCReturn(rc, rc);
580 return VINF_SUCCESS;
581}
582
583
584/**
585 * Reads the VM-entry exception error code field from the VMCS into
586 * the VMX transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600
601/**
602 * Reads the VM-exit interruption-information field from the VMCS into the VMX
603 * transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 */
608DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
611 {
612 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
613 AssertRCReturn(rc, rc);
614 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
615 }
616 return VINF_SUCCESS;
617}
618
619
620/**
621 * Reads the VM-exit interruption error code from the VMCS into the VMX
622 * transient structure.
623 *
624 * @returns VBox status code.
625 * @param pVmxTransient Pointer to the VMX transient structure.
626 */
627DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
630 {
631 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
632 AssertRCReturn(rc, rc);
633 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
634 }
635 return VINF_SUCCESS;
636}
637
638
639/**
640 * Reads the VM-exit instruction length field from the VMCS into the VMX
641 * transient structure.
642 *
643 * @returns VBox status code.
644 * @param pVCpu Pointer to the VMCPU.
645 * @param pVmxTransient Pointer to the VMX transient structure.
646 */
647DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
648{
649 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
650 {
651 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
652 AssertRCReturn(rc, rc);
653 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
654 }
655 return VINF_SUCCESS;
656}
657
658
659/**
660 * Reads the VM-exit instruction-information field from the VMCS into
661 * the VMX transient structure.
662 *
663 * @returns VBox status code.
664 * @param pVmxTransient Pointer to the VMX transient structure.
665 */
666DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
667{
668 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
669 {
670 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
671 AssertRCReturn(rc, rc);
672 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
673 }
674 return VINF_SUCCESS;
675}
676
677
678/**
679 * Reads the exit qualification from the VMCS into the VMX transient structure.
680 *
681 * @returns VBox status code.
682 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
683 * case).
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
689 {
690 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the IDT-vectoring information field from the VMCS into the VMX
700 * transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 *
705 * @remarks No-long-jump zone!!!
706 */
707DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
710 {
711 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the IDT-vectoring error code from the VMCS into the VMX
721 * transient structure.
722 *
723 * @returns VBox status code.
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 */
726DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
727{
728 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
729 {
730 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
731 AssertRCReturn(rc, rc);
732 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
733 }
734 return VINF_SUCCESS;
735}
736
737
738/**
739 * Enters VMX root mode operation on the current CPU.
740 *
741 * @returns VBox status code.
742 * @param pVM Pointer to the VM (optional, can be NULL, after
743 * a resume).
744 * @param HCPhysCpuPage Physical address of the VMXON region.
745 * @param pvCpuPage Pointer to the VMXON region.
746 */
747static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
748{
749 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
750 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
751 Assert(pvCpuPage);
752 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
753
754 if (pVM)
755 {
756 /* Write the VMCS revision dword to the VMXON region. */
757 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
758 }
759
760 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
761 RTCCUINTREG uEflags = ASMIntDisableFlags();
762
763 /* Enable the VMX bit in CR4 if necessary. */
764 RTCCUINTREG uCr4 = ASMGetCR4();
765 if (!(uCr4 & X86_CR4_VMXE))
766 ASMSetCR4(uCr4 | X86_CR4_VMXE);
767
768 /* Enter VMX root mode. */
769 int rc = VMXEnable(HCPhysCpuPage);
770 if (RT_FAILURE(rc))
771 ASMSetCR4(uCr4);
772
773 /* Restore interrupts. */
774 ASMSetFlags(uEflags);
775 return rc;
776}
777
778
779/**
780 * Exits VMX root mode operation on the current CPU.
781 *
782 * @returns VBox status code.
783 */
784static int hmR0VmxLeaveRootMode(void)
785{
786 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
787
788 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
789 RTCCUINTREG uEflags = ASMIntDisableFlags();
790
791 /* If we're for some reason not in VMX root mode, then don't leave it. */
792 RTCCUINTREG uHostCR4 = ASMGetCR4();
793
794 int rc;
795 if (uHostCR4 & X86_CR4_VMXE)
796 {
797 /* Exit VMX root mode and clear the VMX bit in CR4. */
798 VMXDisable();
799 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
800 rc = VINF_SUCCESS;
801 }
802 else
803 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
804
805 /* Restore interrupts. */
806 ASMSetFlags(uEflags);
807 return rc;
808}
809
810
811/**
812 * Allocates and maps one physically contiguous page. The allocated page is
813 * zero'd out. (Used by various VT-x structures).
814 *
815 * @returns IPRT status code.
816 * @param pMemObj Pointer to the ring-0 memory object.
817 * @param ppVirt Where to store the virtual address of the
818 * allocation.
819 * @param pPhys Where to store the physical address of the
820 * allocation.
821 */
822DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
823{
824 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
825 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
826 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
827
828 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
829 if (RT_FAILURE(rc))
830 return rc;
831 *ppVirt = RTR0MemObjAddress(*pMemObj);
832 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
833 ASMMemZero32(*ppVirt, PAGE_SIZE);
834 return VINF_SUCCESS;
835}
836
837
838/**
839 * Frees and unmaps an allocated physical page.
840 *
841 * @param pMemObj Pointer to the ring-0 memory object.
842 * @param ppVirt Where to re-initialize the virtual address of
843 * allocation as 0.
844 * @param pHCPhys Where to re-initialize the physical address of the
845 * allocation as 0.
846 */
847DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
848{
849 AssertPtr(pMemObj);
850 AssertPtr(ppVirt);
851 AssertPtr(pHCPhys);
852 if (*pMemObj != NIL_RTR0MEMOBJ)
853 {
854 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
855 AssertRC(rc);
856 *pMemObj = NIL_RTR0MEMOBJ;
857 *ppVirt = 0;
858 *pHCPhys = 0;
859 }
860}
861
862
863/**
864 * Worker function to free VT-x related structures.
865 *
866 * @returns IPRT status code.
867 * @param pVM Pointer to the VM.
868 */
869static void hmR0VmxStructsFree(PVM pVM)
870{
871 for (VMCPUID i = 0; i < pVM->cCpus; i++)
872 {
873 PVMCPU pVCpu = &pVM->aCpus[i];
874 AssertPtr(pVCpu);
875
876 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
877 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
878
879 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
880 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
881
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
883 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
884 }
885
886 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
887#ifdef VBOX_WITH_CRASHDUMP_MAGIC
888 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
889#endif
890}
891
892
893/**
894 * Worker function to allocate VT-x related VM structures.
895 *
896 * @returns IPRT status code.
897 * @param pVM Pointer to the VM.
898 */
899static int hmR0VmxStructsAlloc(PVM pVM)
900{
901 /*
902 * Initialize members up-front so we can cleanup properly on allocation failure.
903 */
904#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
905 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907 pVM->hm.s.vmx.HCPhys##a_Name = 0;
908
909#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
910 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
911 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
912 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
913
914#ifdef VBOX_WITH_CRASHDUMP_MAGIC
915 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
916#endif
917 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
918
919 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
920 for (VMCPUID i = 0; i < pVM->cCpus; i++)
921 {
922 PVMCPU pVCpu = &pVM->aCpus[i];
923 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
924 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
925 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
926 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
927 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
928 }
929#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
930#undef VMXLOCAL_INIT_VM_MEMOBJ
931
932 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
933 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
934 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
935 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
936
937 /*
938 * Allocate all the VT-x structures.
939 */
940 int rc = VINF_SUCCESS;
941#ifdef VBOX_WITH_CRASHDUMP_MAGIC
942 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
943 if (RT_FAILURE(rc))
944 goto cleanup;
945 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
946 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
947#endif
948
949 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
950 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
951 {
952 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
953 &pVM->hm.s.vmx.HCPhysApicAccess);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956 }
957
958 /*
959 * Initialize per-VCPU VT-x structures.
960 */
961 for (VMCPUID i = 0; i < pVM->cCpus; i++)
962 {
963 PVMCPU pVCpu = &pVM->aCpus[i];
964 AssertPtr(pVCpu);
965
966 /* Allocate the VM control structure (VMCS). */
967 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
968 if (RT_FAILURE(rc))
969 goto cleanup;
970
971 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
972 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
973 {
974 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
975 &pVCpu->hm.s.vmx.HCPhysVirtApic);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978 }
979
980 /*
981 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
982 * transparent accesses of specific MSRs.
983 *
984 * If the condition for enabling MSR bitmaps changes here, don't forget to
985 * update HMIsMsrBitmapsAvailable().
986 */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
988 {
989 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
990 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
994 }
995
996 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
997 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000
1001 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1002 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1003 if (RT_FAILURE(rc))
1004 goto cleanup;
1005 }
1006
1007 return VINF_SUCCESS;
1008
1009cleanup:
1010 hmR0VmxStructsFree(pVM);
1011 return rc;
1012}
1013
1014
1015/**
1016 * Does global VT-x initialization (called during module initialization).
1017 *
1018 * @returns VBox status code.
1019 */
1020VMMR0DECL(int) VMXR0GlobalInit(void)
1021{
1022#ifdef HMVMX_USE_FUNCTION_TABLE
1023 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1024# ifdef VBOX_STRICT
1025 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1026 Assert(g_apfnVMExitHandlers[i]);
1027# endif
1028#endif
1029 return VINF_SUCCESS;
1030}
1031
1032
1033/**
1034 * Does global VT-x termination (called during module termination).
1035 */
1036VMMR0DECL(void) VMXR0GlobalTerm()
1037{
1038 /* Nothing to do currently. */
1039}
1040
1041
1042/**
1043 * Sets up and activates VT-x on the current CPU.
1044 *
1045 * @returns VBox status code.
1046 * @param pCpu Pointer to the global CPU info struct.
1047 * @param pVM Pointer to the VM (can be NULL after a host resume
1048 * operation).
1049 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1050 * fEnabledByHost is true).
1051 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1052 * @a fEnabledByHost is true).
1053 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1054 * enable VT-x on the host.
1055 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1056 */
1057VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1058 void *pvMsrs)
1059{
1060 Assert(pCpu);
1061 Assert(pvMsrs);
1062 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1063
1064 /* Enable VT-x if it's not already enabled by the host. */
1065 if (!fEnabledByHost)
1066 {
1067 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1068 if (RT_FAILURE(rc))
1069 return rc;
1070 }
1071
1072 /*
1073 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1074 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1075 */
1076 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1077 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1078 {
1079 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1080 pCpu->fFlushAsidBeforeUse = false;
1081 }
1082 else
1083 pCpu->fFlushAsidBeforeUse = true;
1084
1085 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1086 ++pCpu->cTlbFlushes;
1087
1088 return VINF_SUCCESS;
1089}
1090
1091
1092/**
1093 * Deactivates VT-x on the current CPU.
1094 *
1095 * @returns VBox status code.
1096 * @param pCpu Pointer to the global CPU info struct.
1097 * @param pvCpuPage Pointer to the VMXON region.
1098 * @param HCPhysCpuPage Physical address of the VMXON region.
1099 *
1100 * @remarks This function should never be called when SUPR0EnableVTx() or
1101 * similar was used to enable VT-x on the host.
1102 */
1103VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1104{
1105 NOREF(pCpu);
1106 NOREF(pvCpuPage);
1107 NOREF(HCPhysCpuPage);
1108
1109 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1110 return hmR0VmxLeaveRootMode();
1111}
1112
1113
1114/**
1115 * Sets the permission bits for the specified MSR in the MSR bitmap.
1116 *
1117 * @param pVCpu Pointer to the VMCPU.
1118 * @param uMSR The MSR value.
1119 * @param enmRead Whether reading this MSR causes a VM-exit.
1120 * @param enmWrite Whether writing this MSR causes a VM-exit.
1121 */
1122static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1123{
1124 int32_t iBit;
1125 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1126
1127 /*
1128 * Layout:
1129 * 0x000 - 0x3ff - Low MSR read bits
1130 * 0x400 - 0x7ff - High MSR read bits
1131 * 0x800 - 0xbff - Low MSR write bits
1132 * 0xc00 - 0xfff - High MSR write bits
1133 */
1134 if (uMsr <= 0x00001FFF)
1135 iBit = uMsr;
1136 else if ( uMsr >= 0xC0000000
1137 && uMsr <= 0xC0001FFF)
1138 {
1139 iBit = (uMsr - 0xC0000000);
1140 pbMsrBitmap += 0x400;
1141 }
1142 else
1143 {
1144 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1145 return;
1146 }
1147
1148 Assert(iBit <= 0x1fff);
1149 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1150 ASMBitSet(pbMsrBitmap, iBit);
1151 else
1152 ASMBitClear(pbMsrBitmap, iBit);
1153
1154 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1155 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1156 else
1157 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1158}
1159
1160
1161#ifdef VBOX_STRICT
1162/**
1163 * Gets the permission bits for the specified MSR in the MSR bitmap.
1164 *
1165 * @returns VBox status code.
1166 * @retval VINF_SUCCESS if the specified MSR is found.
1167 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1168 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1169 *
1170 * @param pVCpu Pointer to the VMCPU.
1171 * @param uMsr The MSR.
1172 * @param penmRead Where to store the read permissions.
1173 * @param penmWrite Where to store the write permissions.
1174 */
1175static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1176{
1177 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1178 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1179 int32_t iBit;
1180 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1181
1182 /* See hmR0VmxSetMsrPermission() for the layout. */
1183 if (uMsr <= 0x00001FFF)
1184 iBit = uMsr;
1185 else if ( uMsr >= 0xC0000000
1186 && uMsr <= 0xC0001FFF)
1187 {
1188 iBit = (uMsr - 0xC0000000);
1189 pbMsrBitmap += 0x400;
1190 }
1191 else
1192 {
1193 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1194 return VERR_NOT_SUPPORTED;
1195 }
1196
1197 Assert(iBit <= 0x1fff);
1198 if (ASMBitTest(pbMsrBitmap, iBit))
1199 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1200 else
1201 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1202
1203 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1204 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1205 else
1206 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1207 return VINF_SUCCESS;
1208}
1209#endif /* VBOX_STRICT */
1210
1211
1212/**
1213 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1214 * area.
1215 *
1216 * @returns VBox status code.
1217 * @param pVCpu Pointer to the VMCPU.
1218 * @param cMsrs The number of MSRs.
1219 */
1220DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1221{
1222 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1223 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1224 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1225 {
1226 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1227 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1228 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1229 }
1230
1231 /* Update number of guest MSRs to load/store across the world-switch. */
1232 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1233 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1234
1235 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1236 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237
1238 /* Update the VCPU's copy of the MSR count. */
1239 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1240
1241 return VINF_SUCCESS;
1242}
1243
1244
1245/**
1246 * Adds a new (or updates the value of an existing) guest/host MSR
1247 * pair to be swapped during the world-switch as part of the
1248 * auto-load/store MSR area in the VMCS.
1249 *
1250 * @returns true if the MSR was added -and- its value was updated, false
1251 * otherwise.
1252 * @param pVCpu Pointer to the VMCPU.
1253 * @param uMsr The MSR.
1254 * @param uGuestMsr Value of the guest MSR.
1255 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1256 * necessary.
1257 */
1258static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1259{
1260 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1262 uint32_t i;
1263 for (i = 0; i < cMsrs; i++)
1264 {
1265 if (pGuestMsr->u32Msr == uMsr)
1266 break;
1267 pGuestMsr++;
1268 }
1269
1270 bool fAdded = false;
1271 if (i == cMsrs)
1272 {
1273 ++cMsrs;
1274 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1275 AssertRC(rc);
1276
1277 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1278 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1279 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1280
1281 fAdded = true;
1282 }
1283
1284 /* Update the MSR values in the auto-load/store MSR area. */
1285 pGuestMsr->u32Msr = uMsr;
1286 pGuestMsr->u64Value = uGuestMsrValue;
1287
1288 /* Create/update the MSR slot in the host MSR area. */
1289 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1290 pHostMsr += i;
1291 pHostMsr->u32Msr = uMsr;
1292
1293 /*
1294 * Update the host MSR only when requested by the caller AND when we're
1295 * adding it to the auto-load/store area. Otherwise, it would have been
1296 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1297 */
1298 bool fUpdatedMsrValue = false;
1299 if ( fAdded
1300 && fUpdateHostMsr)
1301 {
1302 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1303 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1304 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1305 fUpdatedMsrValue = true;
1306 }
1307
1308 return fUpdatedMsrValue;
1309}
1310
1311
1312/**
1313 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1314 * auto-load/store MSR area in the VMCS.
1315 *
1316 * @returns VBox status code.
1317 * @param pVCpu Pointer to the VMCPU.
1318 * @param uMsr The MSR.
1319 */
1320static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1321{
1322 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1323 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1324 for (uint32_t i = 0; i < cMsrs; i++)
1325 {
1326 /* Find the MSR. */
1327 if (pGuestMsr->u32Msr == uMsr)
1328 {
1329 /* If it's the last MSR, simply reduce the count. */
1330 if (i == cMsrs - 1)
1331 {
1332 --cMsrs;
1333 break;
1334 }
1335
1336 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1337 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1338 pLastGuestMsr += cMsrs - 1;
1339 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1340 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1341
1342 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1343 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1344 pLastHostMsr += cMsrs - 1;
1345 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1346 pHostMsr->u64Value = pLastHostMsr->u64Value;
1347 --cMsrs;
1348 break;
1349 }
1350 pGuestMsr++;
1351 }
1352
1353 /* Update the VMCS if the count changed (meaning the MSR was found). */
1354 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1355 {
1356 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1357 AssertRCReturn(rc, rc);
1358
1359 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1360 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1361 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1362
1363 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1364 return VINF_SUCCESS;
1365 }
1366
1367 return VERR_NOT_FOUND;
1368}
1369
1370
1371/**
1372 * Checks if the specified guest MSR is part of the auto-load/store area in
1373 * the VMCS.
1374 *
1375 * @returns true if found, false otherwise.
1376 * @param pVCpu Pointer to the VMCPU.
1377 * @param uMsr The MSR to find.
1378 */
1379static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1380{
1381 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1382 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1383
1384 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1385 {
1386 if (pGuestMsr->u32Msr == uMsr)
1387 return true;
1388 }
1389 return false;
1390}
1391
1392
1393/**
1394 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1395 *
1396 * @param pVCpu Pointer to the VMCPU.
1397 *
1398 * @remarks No-long-jump zone!!!
1399 */
1400static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1401{
1402 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1403 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1405 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1406
1407 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1408 {
1409 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1410
1411 /*
1412 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1413 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1414 */
1415 if (pHostMsr->u32Msr == MSR_K6_EFER)
1416 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1417 else
1418 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1419 }
1420
1421 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1422}
1423
1424
1425#if HC_ARCH_BITS == 64
1426/**
1427 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1428 * perform lazy restoration of the host MSRs while leaving VT-x.
1429 *
1430 * @param pVCpu Pointer to the VMCPU.
1431 *
1432 * @remarks No-long-jump zone!!!
1433 */
1434static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1435{
1436 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1437
1438 /*
1439 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1440 */
1441 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1442 {
1443 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1444 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1445 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1446 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1447 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1448 }
1449}
1450
1451
1452/**
1453 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1454 * lazily while leaving VT-x.
1455 *
1456 * @returns true if it does, false otherwise.
1457 * @param pVCpu Pointer to the VMCPU.
1458 * @param uMsr The MSR to check.
1459 */
1460static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1461{
1462 NOREF(pVCpu);
1463 switch (uMsr)
1464 {
1465 case MSR_K8_LSTAR:
1466 case MSR_K6_STAR:
1467 case MSR_K8_SF_MASK:
1468 case MSR_K8_KERNEL_GS_BASE:
1469 return true;
1470 }
1471 return false;
1472}
1473
1474
1475/**
1476 * Saves a set of guests MSRs back into the guest-CPU context.
1477 *
1478 * @param pVCpu Pointer to the VMCPU.
1479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1480 * out-of-sync. Make sure to update the required fields
1481 * before using them.
1482 *
1483 * @remarks No-long-jump zone!!!
1484 */
1485static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1486{
1487 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1488 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1489
1490 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1491 {
1492 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1493 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1494 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1495 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1496 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1497 }
1498}
1499
1500
1501/**
1502 * Loads a set of guests MSRs to allow read/passthru to the guest.
1503 *
1504 * The name of this function is slightly confusing. This function does NOT
1505 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1506 * common prefix for functions dealing with "lazy restoration" of the shared
1507 * MSRs.
1508 *
1509 * @param pVCpu Pointer to the VMCPU.
1510 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1511 * out-of-sync. Make sure to update the required fields
1512 * before using them.
1513 *
1514 * @remarks No-long-jump zone!!!
1515 */
1516static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1517{
1518 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1519 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1520
1521 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1522 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1523 {
1524#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1525 do { \
1526 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1527 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1528 else \
1529 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1530 } while (0)
1531
1532 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1533 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1534 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1535 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1536#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1537 }
1538 else
1539 {
1540 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1541 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1542 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1543 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1544 }
1545 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1546}
1547
1548
1549/**
1550 * Performs lazy restoration of the set of host MSRs if they were previously
1551 * loaded with guest MSR values.
1552 *
1553 * @param pVCpu Pointer to the VMCPU.
1554 *
1555 * @remarks No-long-jump zone!!!
1556 * @remarks The guest MSRs should have been saved back into the guest-CPU
1557 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1558 */
1559static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1560{
1561 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1562 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1563
1564 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1565 {
1566 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1567 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1568 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1569 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1570 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1571 }
1572 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1573}
1574#endif /* HC_ARCH_BITS == 64 */
1575
1576
1577/**
1578 * Verifies that our cached values of the VMCS controls are all
1579 * consistent with what's actually present in the VMCS.
1580 *
1581 * @returns VBox status code.
1582 * @param pVCpu Pointer to the VMCPU.
1583 */
1584static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1585{
1586 uint32_t u32Val;
1587 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1588 AssertRCReturn(rc, rc);
1589 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1590 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1591
1592 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1593 AssertRCReturn(rc, rc);
1594 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1595 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1596
1597 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1598 AssertRCReturn(rc, rc);
1599 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1600 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1601
1602 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1603 AssertRCReturn(rc, rc);
1604 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1605 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1606
1607 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1608 AssertRCReturn(rc, rc);
1609 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1610 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1611
1612 return VINF_SUCCESS;
1613}
1614
1615
1616#ifdef VBOX_STRICT
1617/**
1618 * Verifies that our cached host EFER value has not changed
1619 * since we cached it.
1620 *
1621 * @param pVCpu Pointer to the VMCPU.
1622 */
1623static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1624{
1625 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1626
1627 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1628 {
1629 uint64_t u64Val;
1630 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1631 AssertRC(rc);
1632
1633 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1634 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1635 }
1636}
1637
1638
1639/**
1640 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1641 * VMCS are correct.
1642 *
1643 * @param pVCpu Pointer to the VMCPU.
1644 */
1645static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1646{
1647 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1648
1649 /* Verify MSR counts in the VMCS are what we think it should be. */
1650 uint32_t cMsrs;
1651 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1652 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1653
1654 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1655 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1656
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1658 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1659
1660 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1661 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1662 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1663 {
1664 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1665 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1666 pGuestMsr->u32Msr, cMsrs));
1667
1668 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1669 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1670 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1671
1672 /* Verify that the permissions are as expected in the MSR bitmap. */
1673 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1674 {
1675 VMXMSREXITREAD enmRead;
1676 VMXMSREXITWRITE enmWrite;
1677 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1678 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1679 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1680 {
1681 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1682 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1683 }
1684 else
1685 {
1686 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1687 pGuestMsr->u32Msr, cMsrs));
1688 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1689 pGuestMsr->u32Msr, cMsrs));
1690 }
1691 }
1692 }
1693}
1694#endif /* VBOX_STRICT */
1695
1696
1697/**
1698 * Flushes the TLB using EPT.
1699 *
1700 * @returns VBox status code.
1701 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1702 * enmFlush).
1703 * @param enmFlush Type of flush.
1704 *
1705 * @remarks Caller is responsible for making sure this function is called only
1706 * when NestedPaging is supported and providing @a enmFlush that is
1707 * supported by the CPU.
1708 * @remarks Can be called with interrupts disabled.
1709 */
1710static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1711{
1712 uint64_t au64Descriptor[2];
1713 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1714 au64Descriptor[0] = 0;
1715 else
1716 {
1717 Assert(pVCpu);
1718 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1719 }
1720 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1721
1722 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1723 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1724 rc));
1725 if ( RT_SUCCESS(rc)
1726 && pVCpu)
1727 {
1728 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1729 }
1730}
1731
1732
1733/**
1734 * Flushes the TLB using VPID.
1735 *
1736 * @returns VBox status code.
1737 * @param pVM Pointer to the VM.
1738 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1739 * enmFlush).
1740 * @param enmFlush Type of flush.
1741 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1742 * on @a enmFlush).
1743 *
1744 * @remarks Can be called with interrupts disabled.
1745 */
1746static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1747{
1748 NOREF(pVM);
1749 AssertPtr(pVM);
1750 Assert(pVM->hm.s.vmx.fVpid);
1751
1752 uint64_t au64Descriptor[2];
1753 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1754 {
1755 au64Descriptor[0] = 0;
1756 au64Descriptor[1] = 0;
1757 }
1758 else
1759 {
1760 AssertPtr(pVCpu);
1761 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1762 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1763 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1764 au64Descriptor[1] = GCPtr;
1765 }
1766
1767 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1768 AssertMsg(rc == VINF_SUCCESS,
1769 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1770 if ( RT_SUCCESS(rc)
1771 && pVCpu)
1772 {
1773 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1774 }
1775}
1776
1777
1778/**
1779 * Invalidates a guest page by guest virtual address. Only relevant for
1780 * EPT/VPID, otherwise there is nothing really to invalidate.
1781 *
1782 * @returns VBox status code.
1783 * @param pVM Pointer to the VM.
1784 * @param pVCpu Pointer to the VMCPU.
1785 * @param GCVirt Guest virtual address of the page to invalidate.
1786 */
1787VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1788{
1789 AssertPtr(pVM);
1790 AssertPtr(pVCpu);
1791 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1792
1793 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1794 if (!fFlushPending)
1795 {
1796 /*
1797 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1798 * See @bugref{6043} and @bugref{6177}.
1799 *
1800 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1801 * function maybe called in a loop with individual addresses.
1802 */
1803 if (pVM->hm.s.vmx.fVpid)
1804 {
1805 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1806 {
1807 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1808 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1809 }
1810 else
1811 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1812 }
1813 else if (pVM->hm.s.fNestedPaging)
1814 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1815 }
1816
1817 return VINF_SUCCESS;
1818}
1819
1820
1821/**
1822 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1823 * otherwise there is nothing really to invalidate.
1824 *
1825 * @returns VBox status code.
1826 * @param pVM Pointer to the VM.
1827 * @param pVCpu Pointer to the VMCPU.
1828 * @param GCPhys Guest physical address of the page to invalidate.
1829 */
1830VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1831{
1832 NOREF(pVM); NOREF(GCPhys);
1833 LogFlowFunc(("%RGp\n", GCPhys));
1834
1835 /*
1836 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1837 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1838 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1839 */
1840 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1841 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1842 return VINF_SUCCESS;
1843}
1844
1845
1846/**
1847 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1848 * case where neither EPT nor VPID is supported by the CPU.
1849 *
1850 * @param pVM Pointer to the VM.
1851 * @param pVCpu Pointer to the VMCPU.
1852 * @param pCpu Pointer to the global HM struct.
1853 *
1854 * @remarks Called with interrupts disabled.
1855 */
1856static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1857{
1858 AssertPtr(pVCpu);
1859 AssertPtr(pCpu);
1860 NOREF(pVM);
1861
1862 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1863
1864 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1865#if 0
1866 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1867 pVCpu->hm.s.TlbShootdown.cPages = 0;
1868#endif
1869
1870 Assert(pCpu->idCpu != NIL_RTCPUID);
1871 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1872 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1873 pVCpu->hm.s.fForceTLBFlush = false;
1874 return;
1875}
1876
1877
1878/**
1879 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1880 *
1881 * @param pVM Pointer to the VM.
1882 * @param pVCpu Pointer to the VMCPU.
1883 * @param pCpu Pointer to the global HM CPU struct.
1884 * @remarks All references to "ASID" in this function pertains to "VPID" in
1885 * Intel's nomenclature. The reason is, to avoid confusion in compare
1886 * statements since the host-CPU copies are named "ASID".
1887 *
1888 * @remarks Called with interrupts disabled.
1889 */
1890static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1891{
1892#ifdef VBOX_WITH_STATISTICS
1893 bool fTlbFlushed = false;
1894# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1895# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1896 if (!fTlbFlushed) \
1897 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1898 } while (0)
1899#else
1900# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1901# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1902#endif
1903
1904 AssertPtr(pVM);
1905 AssertPtr(pCpu);
1906 AssertPtr(pVCpu);
1907 Assert(pCpu->idCpu != NIL_RTCPUID);
1908
1909 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1910 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1911 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1912
1913 /*
1914 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1915 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1916 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1917 */
1918 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1919 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1920 {
1921 ++pCpu->uCurrentAsid;
1922 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1923 {
1924 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1925 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1926 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1927 }
1928
1929 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1930 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1931 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1932
1933 /*
1934 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1935 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1936 */
1937 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1938 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1939 HMVMX_SET_TAGGED_TLB_FLUSHED();
1940 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1941 }
1942
1943 /* Check for explicit TLB shootdowns. */
1944 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1945 {
1946 /*
1947 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1948 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1949 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1950 * but not guest-physical mappings.
1951 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1952 */
1953 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1955 HMVMX_SET_TAGGED_TLB_FLUSHED();
1956 }
1957
1958 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1959 * where it is commented out. Support individual entry flushing
1960 * someday. */
1961#if 0
1962 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1963 {
1964 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965
1966 /*
1967 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1968 * as supported by the CPU.
1969 */
1970 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1971 {
1972 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1973 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1974 }
1975 else
1976 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1977
1978 HMVMX_SET_TAGGED_TLB_FLUSHED();
1979 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1980 pVCpu->hm.s.TlbShootdown.cPages = 0;
1981 }
1982#endif
1983
1984 pVCpu->hm.s.fForceTLBFlush = false;
1985
1986 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1987
1988 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1989 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1990 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1991 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1992 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1993 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1994 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1995 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1996 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1997
1998 /* Update VMCS with the VPID. */
1999 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2000 AssertRC(rc);
2001
2002#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2003}
2004
2005
2006/**
2007 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2008 *
2009 * @returns VBox status code.
2010 * @param pVM Pointer to the VM.
2011 * @param pVCpu Pointer to the VMCPU.
2012 * @param pCpu Pointer to the global HM CPU struct.
2013 *
2014 * @remarks Called with interrupts disabled.
2015 */
2016static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2017{
2018 AssertPtr(pVM);
2019 AssertPtr(pVCpu);
2020 AssertPtr(pCpu);
2021 Assert(pCpu->idCpu != NIL_RTCPUID);
2022 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2023 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2024
2025 /*
2026 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2027 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2028 */
2029 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2030 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2031 {
2032 pVCpu->hm.s.fForceTLBFlush = true;
2033 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034 }
2035
2036 /* Check for explicit TLB shootdown flushes. */
2037 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2038 {
2039 pVCpu->hm.s.fForceTLBFlush = true;
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2041 }
2042
2043 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2044 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2045
2046 if (pVCpu->hm.s.fForceTLBFlush)
2047 {
2048 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2049 pVCpu->hm.s.fForceTLBFlush = false;
2050 }
2051 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2052 * where it is commented out. Support individual entry flushing
2053 * someday. */
2054#if 0
2055 else
2056 {
2057 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2058 {
2059 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2060 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2061 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2062 }
2063 else
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2065
2066 pVCpu->hm.s.TlbShootdown.cPages = 0;
2067 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2068 }
2069#endif
2070}
2071
2072
2073/**
2074 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2075 *
2076 * @returns VBox status code.
2077 * @param pVM Pointer to the VM.
2078 * @param pVCpu Pointer to the VMCPU.
2079 * @param pCpu Pointer to the global HM CPU struct.
2080 *
2081 * @remarks Called with interrupts disabled.
2082 */
2083static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2084{
2085 AssertPtr(pVM);
2086 AssertPtr(pVCpu);
2087 AssertPtr(pCpu);
2088 Assert(pCpu->idCpu != NIL_RTCPUID);
2089 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2090 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2091
2092 /*
2093 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2094 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2095 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2096 */
2097 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2098 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2099 {
2100 pVCpu->hm.s.fForceTLBFlush = true;
2101 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2102 }
2103
2104 /* Check for explicit TLB shootdown flushes. */
2105 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2106 {
2107 /*
2108 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2109 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2110 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2111 */
2112 pVCpu->hm.s.fForceTLBFlush = true;
2113 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2114 }
2115
2116 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2117 if (pVCpu->hm.s.fForceTLBFlush)
2118 {
2119 ++pCpu->uCurrentAsid;
2120 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2121 {
2122 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2123 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2124 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2125 }
2126
2127 pVCpu->hm.s.fForceTLBFlush = false;
2128 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2129 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2130 if (pCpu->fFlushAsidBeforeUse)
2131 {
2132 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2133 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2134 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2135 {
2136 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2137 pCpu->fFlushAsidBeforeUse = false;
2138 }
2139 else
2140 {
2141 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2142 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2143 }
2144 }
2145 }
2146 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2147 * where it is commented out. Support individual entry flushing
2148 * someday. */
2149#if 0
2150 else
2151 {
2152 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2153 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2154 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2155 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2156
2157 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2158 {
2159 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2160 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2161 {
2162 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2163 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2164 }
2165 else
2166 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2167
2168 pVCpu->hm.s.TlbShootdown.cPages = 0;
2169 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2170 }
2171 else
2172 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2173 }
2174#endif
2175
2176 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2177 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2178 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2179 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2180 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2181 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2182 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2183
2184 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2185 AssertRC(rc);
2186}
2187
2188
2189/**
2190 * Flushes the guest TLB entry based on CPU capabilities.
2191 *
2192 * @param pVCpu Pointer to the VMCPU.
2193 * @param pCpu Pointer to the global HM CPU struct.
2194 */
2195DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2196{
2197#ifdef HMVMX_ALWAYS_FLUSH_TLB
2198 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2199#endif
2200 PVM pVM = pVCpu->CTX_SUFF(pVM);
2201 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2202 {
2203 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2204 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2205 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2206 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2207 default:
2208 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2209 break;
2210 }
2211
2212 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2213 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2214
2215 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2216}
2217
2218
2219/**
2220 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2221 * TLB entries from the host TLB before VM-entry.
2222 *
2223 * @returns VBox status code.
2224 * @param pVM Pointer to the VM.
2225 */
2226static int hmR0VmxSetupTaggedTlb(PVM pVM)
2227{
2228 /*
2229 * Determine optimal flush type for Nested Paging.
2230 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2231 * guest execution (see hmR3InitFinalizeR0()).
2232 */
2233 if (pVM->hm.s.fNestedPaging)
2234 {
2235 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2236 {
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2238 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2239 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2240 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2241 else
2242 {
2243 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2244 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2245 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2246 }
2247
2248 /* Make sure the write-back cacheable memory type for EPT is supported. */
2249 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2250 {
2251 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2252 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2253 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2254 }
2255 }
2256 else
2257 {
2258 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2259 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2261 }
2262 }
2263
2264 /*
2265 * Determine optimal flush type for VPID.
2266 */
2267 if (pVM->hm.s.vmx.fVpid)
2268 {
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2270 {
2271 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2272 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2273 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2274 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2275 else
2276 {
2277 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2278 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2279 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2280 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2281 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2282 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2283 pVM->hm.s.vmx.fVpid = false;
2284 }
2285 }
2286 else
2287 {
2288 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2289 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2290 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2291 pVM->hm.s.vmx.fVpid = false;
2292 }
2293 }
2294
2295 /*
2296 * Setup the handler for flushing tagged-TLBs.
2297 */
2298 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2299 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2300 else if (pVM->hm.s.fNestedPaging)
2301 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2302 else if (pVM->hm.s.vmx.fVpid)
2303 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2304 else
2305 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2306 return VINF_SUCCESS;
2307}
2308
2309
2310/**
2311 * Sets up pin-based VM-execution controls in the VMCS.
2312 *
2313 * @returns VBox status code.
2314 * @param pVM Pointer to the VM.
2315 * @param pVCpu Pointer to the VMCPU.
2316 */
2317static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2318{
2319 AssertPtr(pVM);
2320 AssertPtr(pVCpu);
2321
2322 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2323 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2324
2325 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2326 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2327
2328 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2329 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2330
2331 /* Enable the VMX preemption timer. */
2332 if (pVM->hm.s.vmx.fUsePreemptTimer)
2333 {
2334 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2335 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2336 }
2337
2338 if ((val & zap) != val)
2339 {
2340 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2341 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2342 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2343 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2344 }
2345
2346 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2347 AssertRCReturn(rc, rc);
2348
2349 /* Update VCPU with the currently set pin-based VM-execution controls. */
2350 pVCpu->hm.s.vmx.u32PinCtls = val;
2351 return rc;
2352}
2353
2354
2355/**
2356 * Sets up processor-based VM-execution controls in the VMCS.
2357 *
2358 * @returns VBox status code.
2359 * @param pVM Pointer to the VM.
2360 * @param pVMCPU Pointer to the VMCPU.
2361 */
2362static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2363{
2364 AssertPtr(pVM);
2365 AssertPtr(pVCpu);
2366
2367 int rc = VERR_INTERNAL_ERROR_5;
2368 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2378
2379 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2380 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2381 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2382 {
2383 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2384 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2385 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2386 }
2387
2388 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2389 if (!pVM->hm.s.fNestedPaging)
2390 {
2391 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2392 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2393 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2394 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2395 }
2396
2397 /* Use TPR shadowing if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2399 {
2400 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2401 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2402 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2403 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2404 AssertRCReturn(rc, rc);
2405
2406 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2407 /* CR8 writes cause a VM-exit based on TPR threshold. */
2408 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2409 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2410 }
2411 else
2412 {
2413 /*
2414 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2415 * Set this control only for 64-bit guests.
2416 */
2417 if (pVM->hm.s.fAllow64BitGuests)
2418 {
2419 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2421 }
2422 }
2423
2424 /* Use MSR-bitmaps if supported by the CPU. */
2425 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2426 {
2427 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2428
2429 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2431 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2432 AssertRCReturn(rc, rc);
2433
2434 /*
2435 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2436 * automatically using dedicated fields in the VMCS.
2437 */
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443
2444#if HC_ARCH_BITS == 64
2445 /*
2446 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2447 */
2448 if (pVM->hm.s.fAllow64BitGuests)
2449 {
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454 }
2455#endif
2456 }
2457
2458 /* If we're using virtual NMIs, we need the NMI-window exiting feature. */
2459 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2460 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
2461 {
2462 val |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
2463 }
2464
2465 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2466 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2467 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2468
2469 if ((val & zap) != val)
2470 {
2471 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2472 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2473 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2474 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2475 }
2476
2477 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2478 AssertRCReturn(rc, rc);
2479
2480 /* Update VCPU with the currently set processor-based VM-execution controls. */
2481 pVCpu->hm.s.vmx.u32ProcCtls = val;
2482
2483 /*
2484 * Secondary processor-based VM-execution controls.
2485 */
2486 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2487 {
2488 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2489 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2490
2491 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2492 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2493
2494 if (pVM->hm.s.fNestedPaging)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2496 else
2497 {
2498 /*
2499 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2500 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2501 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2502 */
2503 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2505 }
2506
2507 if (pVM->hm.s.vmx.fVpid)
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2509
2510 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2511 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ((val & zap) != val)
2529 {
2530 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2531 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2532 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2533 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2534 }
2535
2536 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2537 AssertRCReturn(rc, rc);
2538
2539 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2540 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2541 }
2542 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2543 {
2544 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2545 "available\n"));
2546 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2547 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2548 }
2549
2550 return VINF_SUCCESS;
2551}
2552
2553
2554/**
2555 * Sets up miscellaneous (everything other than Pin & Processor-based
2556 * VM-execution) control fields in the VMCS.
2557 *
2558 * @returns VBox status code.
2559 * @param pVM Pointer to the VM.
2560 * @param pVCpu Pointer to the VMCPU.
2561 */
2562static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2563{
2564 NOREF(pVM);
2565 AssertPtr(pVM);
2566 AssertPtr(pVCpu);
2567
2568 int rc = VERR_GENERAL_FAILURE;
2569
2570 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2571#if 0
2572 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2573 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2575
2576 /*
2577 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2578 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2579 * We thus use the exception bitmap to control it rather than use both.
2580 */
2581 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2582 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2583
2584 /** @todo Explore possibility of using IO-bitmaps. */
2585 /* All IO & IOIO instructions cause VM-exits. */
2586 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2587 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2588
2589 /* Initialize the MSR-bitmap area. */
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2592 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2593#endif
2594
2595 /* Setup MSR auto-load/store area. */
2596 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2597 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2598 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2599 AssertRCReturn(rc, rc);
2600 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2601 AssertRCReturn(rc, rc);
2602
2603 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2604 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2605 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2606 AssertRCReturn(rc, rc);
2607
2608 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2610 AssertRCReturn(rc, rc);
2611
2612 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2613#if 0
2614 /* Setup debug controls */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2616 AssertRCReturn(rc, rc);
2617 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2618 AssertRCReturn(rc, rc);
2619#endif
2620
2621 return rc;
2622}
2623
2624
2625/**
2626 * Sets up the initial exception bitmap in the VMCS based on static conditions
2627 * (i.e. conditions that cannot ever change after starting the VM).
2628 *
2629 * @returns VBox status code.
2630 * @param pVM Pointer to the VM.
2631 * @param pVCpu Pointer to the VMCPU.
2632 */
2633static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2634{
2635 AssertPtr(pVM);
2636 AssertPtr(pVCpu);
2637
2638 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2639
2640 uint32_t u32XcptBitmap = 0;
2641
2642 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2643 if (!pVM->hm.s.fNestedPaging)
2644 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2645
2646 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2647 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2648 AssertRCReturn(rc, rc);
2649 return rc;
2650}
2651
2652
2653/**
2654 * Sets up the initial guest-state mask. The guest-state mask is consulted
2655 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2656 * for the nested virtualization case (as it would cause a VM-exit).
2657 *
2658 * @param pVCpu Pointer to the VMCPU.
2659 */
2660static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2661{
2662 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2663 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2664 return VINF_SUCCESS;
2665}
2666
2667
2668/**
2669 * Does per-VM VT-x initialization.
2670 *
2671 * @returns VBox status code.
2672 * @param pVM Pointer to the VM.
2673 */
2674VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2675{
2676 LogFlowFunc(("pVM=%p\n", pVM));
2677
2678 int rc = hmR0VmxStructsAlloc(pVM);
2679 if (RT_FAILURE(rc))
2680 {
2681 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2682 return rc;
2683 }
2684
2685 return VINF_SUCCESS;
2686}
2687
2688
2689/**
2690 * Does per-VM VT-x termination.
2691 *
2692 * @returns VBox status code.
2693 * @param pVM Pointer to the VM.
2694 */
2695VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2696{
2697 LogFlowFunc(("pVM=%p\n", pVM));
2698
2699#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2700 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2701 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2702#endif
2703 hmR0VmxStructsFree(pVM);
2704 return VINF_SUCCESS;
2705}
2706
2707
2708/**
2709 * Sets up the VM for execution under VT-x.
2710 * This function is only called once per-VM during initialization.
2711 *
2712 * @returns VBox status code.
2713 * @param pVM Pointer to the VM.
2714 */
2715VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2716{
2717 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2718 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2719
2720 LogFlowFunc(("pVM=%p\n", pVM));
2721
2722 /*
2723 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2724 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2725 */
2726 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2727 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2728 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2729 || !pVM->hm.s.vmx.pRealModeTSS))
2730 {
2731 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2732 return VERR_INTERNAL_ERROR;
2733 }
2734
2735#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2736 /*
2737 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2738 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2739 */
2740 if ( pVM->hm.s.fAllow64BitGuests
2741 && !HMVMX_IS_64BIT_HOST_MODE())
2742 {
2743 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2744 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2745 }
2746#endif
2747
2748 /* Initialize these always, see hmR3InitFinalizeR0().*/
2749 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2750 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2751
2752 /* Setup the tagged-TLB flush handlers. */
2753 int rc = hmR0VmxSetupTaggedTlb(pVM);
2754 if (RT_FAILURE(rc))
2755 {
2756 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2757 return rc;
2758 }
2759
2760 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2761 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2762#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2763 if ( HMVMX_IS_64BIT_HOST_MODE()
2764 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2765 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2767 {
2768 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2769 }
2770#endif
2771
2772 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2773 {
2774 PVMCPU pVCpu = &pVM->aCpus[i];
2775 AssertPtr(pVCpu);
2776 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2777
2778 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2779 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2780
2781 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2782 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2783 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2784
2785 /* Set revision dword at the beginning of the VMCS structure. */
2786 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2787
2788 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2789 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2790 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2791 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2792
2793 /* Load this VMCS as the current VMCS. */
2794 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797
2798 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2800 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2801
2802 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2815 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2819 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2820 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2821 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2822#endif
2823
2824 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2825 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828
2829 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2830
2831 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2832 }
2833
2834 return VINF_SUCCESS;
2835}
2836
2837
2838/**
2839 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2840 * the VMCS.
2841 *
2842 * @returns VBox status code.
2843 * @param pVM Pointer to the VM.
2844 * @param pVCpu Pointer to the VMCPU.
2845 */
2846DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2847{
2848 NOREF(pVM); NOREF(pVCpu);
2849
2850 RTCCUINTREG uReg = ASMGetCR0();
2851 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2852 AssertRCReturn(rc, rc);
2853
2854#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2855 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2856 if (HMVMX_IS_64BIT_HOST_MODE())
2857 {
2858 uint64_t uRegCR3 = HMR0Get64bitCR3();
2859 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2860 }
2861 else
2862#endif
2863 {
2864 uReg = ASMGetCR3();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2866 }
2867 AssertRCReturn(rc, rc);
2868
2869 uReg = ASMGetCR4();
2870 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2871 AssertRCReturn(rc, rc);
2872 return rc;
2873}
2874
2875
2876#if HC_ARCH_BITS == 64
2877/**
2878 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2879 * requirements. See hmR0VmxSaveHostSegmentRegs().
2880 */
2881# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2882 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2883 { \
2884 bool fValidSelector = true; \
2885 if ((selValue) & X86_SEL_LDT) \
2886 { \
2887 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2888 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2889 } \
2890 if (fValidSelector) \
2891 { \
2892 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2893 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2894 } \
2895 (selValue) = 0; \
2896 }
2897#endif
2898
2899
2900/**
2901 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2902 * the host-state area in the VMCS.
2903 *
2904 * @returns VBox status code.
2905 * @param pVM Pointer to the VM.
2906 * @param pVCpu Pointer to the VMCPU.
2907 */
2908DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2909{
2910 NOREF(pVM);
2911 int rc = VERR_INTERNAL_ERROR_5;
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2916 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2917 */
2918 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2919 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2920#endif
2921
2922 /*
2923 * Host DS, ES, FS and GS segment registers.
2924 */
2925#if HC_ARCH_BITS == 64
2926 RTSEL uSelDS = ASMGetDS();
2927 RTSEL uSelES = ASMGetES();
2928 RTSEL uSelFS = ASMGetFS();
2929 RTSEL uSelGS = ASMGetGS();
2930#else
2931 RTSEL uSelDS = 0;
2932 RTSEL uSelES = 0;
2933 RTSEL uSelFS = 0;
2934 RTSEL uSelGS = 0;
2935#endif
2936
2937 /* Recalculate which host-state bits need to be manually restored. */
2938 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2939
2940 /*
2941 * Host CS and SS segment registers.
2942 */
2943#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2944 RTSEL uSelCS;
2945 RTSEL uSelSS;
2946 if (HMVMX_IS_64BIT_HOST_MODE())
2947 {
2948 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2949 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2950 }
2951 else
2952 {
2953 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2954 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2955 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2956 }
2957#else
2958 RTSEL uSelCS = ASMGetCS();
2959 RTSEL uSelSS = ASMGetSS();
2960#endif
2961
2962 /*
2963 * Host TR segment register.
2964 */
2965 RTSEL uSelTR = ASMGetTR();
2966
2967#if HC_ARCH_BITS == 64
2968 /*
2969 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2970 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2971 */
2972 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2973 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2974 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2975 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2976# undef VMXLOCAL_ADJUST_HOST_SEG
2977#endif
2978
2979 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2980 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2981 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2982 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2983 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2984 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2985 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2986 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2987 Assert(uSelCS);
2988 Assert(uSelTR);
2989
2990 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2991#if 0
2992 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2993 Assert(uSelSS != 0);
2994#endif
2995
2996 /* Write these host selector fields into the host-state area in the VMCS. */
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2999#if HC_ARCH_BITS == 64
3000 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
3001 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
3002 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
3003 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
3004#endif
3005 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3006
3007 /*
3008 * Host GDTR and IDTR.
3009 */
3010 RTGDTR Gdtr;
3011 RT_ZERO(Gdtr);
3012#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3013 if (HMVMX_IS_64BIT_HOST_MODE())
3014 {
3015 X86XDTR64 Gdtr64;
3016 X86XDTR64 Idtr64;
3017 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3018 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3019 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3020
3021 Gdtr.cbGdt = Gdtr64.cb;
3022 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3023 }
3024 else
3025#endif
3026 {
3027 RTIDTR Idtr;
3028 ASMGetGDTR(&Gdtr);
3029 ASMGetIDTR(&Idtr);
3030 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3031 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3032
3033#if HC_ARCH_BITS == 64
3034 /*
3035 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3036 * maximum limit (0xffff) on every VM-exit.
3037 */
3038 if (Gdtr.cbGdt != 0xffff)
3039 {
3040 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3041 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3042 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3043 }
3044
3045 /*
3046 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3047 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3048 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3049 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3050 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3051 * hosts where we are pretty sure it won't cause trouble.
3052 */
3053# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3054 if (Idtr.cbIdt < 0x0fff)
3055# else
3056 if (Idtr.cbIdt != 0xffff)
3057# endif
3058 {
3059 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3060 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3061 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3062 }
3063#endif
3064 }
3065
3066 /*
3067 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3068 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3069 */
3070 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3071 {
3072 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3073 return VERR_VMX_INVALID_HOST_STATE;
3074 }
3075
3076 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3077#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3078 if (HMVMX_IS_64BIT_HOST_MODE())
3079 {
3080 /* We need the 64-bit TR base for hybrid darwin. */
3081 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3082 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3083 }
3084 else
3085#endif
3086 {
3087 uintptr_t uTRBase;
3088#if HC_ARCH_BITS == 64
3089 uTRBase = X86DESC64_BASE(pDesc);
3090
3091 /*
3092 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3093 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3094 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3095 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3096 *
3097 * [1] See Intel spec. 3.5 "System Descriptor Types".
3098 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3099 */
3100 Assert(pDesc->System.u4Type == 11);
3101 if ( pDesc->System.u16LimitLow != 0x67
3102 || pDesc->System.u4LimitHigh)
3103 {
3104 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3105 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3106
3107 /* Store the GDTR here as we need it while restoring TR. */
3108 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3109 }
3110#else
3111 uTRBase = X86DESC_BASE(pDesc);
3112#endif
3113 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3114 }
3115 AssertRCReturn(rc, rc);
3116
3117 /*
3118 * Host FS base and GS base.
3119 */
3120#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3121 if (HMVMX_IS_64BIT_HOST_MODE())
3122 {
3123 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3124 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3125 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3126 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3127
3128# if HC_ARCH_BITS == 64
3129 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3130 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3131 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3132 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3133 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3134# endif
3135 }
3136#endif
3137 return rc;
3138}
3139
3140
3141/**
3142 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3143 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3144 * the host after every successful VM-exit.
3145 *
3146 * @returns VBox status code.
3147 * @param pVM Pointer to the VM.
3148 * @param pVCpu Pointer to the VMCPU.
3149 *
3150 * @remarks No-long-jump zone!!!
3151 */
3152DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3153{
3154 NOREF(pVM);
3155
3156 AssertPtr(pVCpu);
3157 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3158
3159 int rc = VINF_SUCCESS;
3160#if HC_ARCH_BITS == 64
3161 if (pVM->hm.s.fAllow64BitGuests)
3162 hmR0VmxLazySaveHostMsrs(pVCpu);
3163#endif
3164
3165 /*
3166 * Host Sysenter MSRs.
3167 */
3168 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3169 AssertRCReturn(rc, rc);
3170#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3171 if (HMVMX_IS_64BIT_HOST_MODE())
3172 {
3173 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3174 AssertRCReturn(rc, rc);
3175 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3176 }
3177 else
3178 {
3179 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3180 AssertRCReturn(rc, rc);
3181 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3182 }
3183#elif HC_ARCH_BITS == 32
3184 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3185 AssertRCReturn(rc, rc);
3186 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3187#else
3188 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3189 AssertRCReturn(rc, rc);
3190 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3191#endif
3192 AssertRCReturn(rc, rc);
3193
3194 /*
3195 * Host EFER MSR.
3196 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3197 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3198 */
3199 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3200 {
3201 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3202 AssertRCReturn(rc, rc);
3203 }
3204
3205 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3206 * hmR0VmxLoadGuestExitCtls() !! */
3207
3208 return rc;
3209}
3210
3211
3212/**
3213 * Figures out if we need to swap the EFER MSR which is
3214 * particularly expensive.
3215 *
3216 * We check all relevant bits. For now, that's everything
3217 * besides LMA/LME, as these two bits are handled by VM-entry,
3218 * see hmR0VmxLoadGuestExitCtls() and
3219 * hmR0VMxLoadGuestEntryCtls().
3220 *
3221 * @returns true if we need to load guest EFER, false otherwise.
3222 * @param pVCpu Pointer to the VMCPU.
3223 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3224 * out-of-sync. Make sure to update the required fields
3225 * before using them.
3226 *
3227 * @remarks Requires EFER, CR4.
3228 * @remarks No-long-jump zone!!!
3229 */
3230static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3231{
3232#ifdef HMVMX_ALWAYS_SWAP_EFER
3233 return true;
3234#endif
3235
3236#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3237 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3238 if (CPUMIsGuestInLongMode(pVCpu))
3239 return false;
3240#endif
3241
3242 PVM pVM = pVCpu->CTX_SUFF(pVM);
3243 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3244 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3245
3246 /*
3247 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3248 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3249 */
3250 if ( CPUMIsGuestInLongMode(pVCpu)
3251 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3252 {
3253 return true;
3254 }
3255
3256 /*
3257 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3258 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3259 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3260 */
3261 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3262 && (pMixedCtx->cr0 & X86_CR0_PG)
3263 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3264 {
3265 /* Assert that host is PAE capable. */
3266 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3267 return true;
3268 }
3269
3270 /** @todo Check the latest Intel spec. for any other bits,
3271 * like SMEP/SMAP? */
3272 return false;
3273}
3274
3275
3276/**
3277 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3278 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3279 * controls".
3280 *
3281 * @returns VBox status code.
3282 * @param pVCpu Pointer to the VMCPU.
3283 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3284 * out-of-sync. Make sure to update the required fields
3285 * before using them.
3286 *
3287 * @remarks Requires EFER.
3288 * @remarks No-long-jump zone!!!
3289 */
3290DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3291{
3292 int rc = VINF_SUCCESS;
3293 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3294 {
3295 PVM pVM = pVCpu->CTX_SUFF(pVM);
3296 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3297 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3298
3299 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3300 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3301
3302 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3303 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3304 {
3305 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3306 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3307 }
3308 else
3309 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3310
3311 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3312 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3313 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3314 {
3315 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3316 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3317 }
3318
3319 /*
3320 * The following should -not- be set (since we're not in SMM mode):
3321 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3322 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3323 */
3324
3325 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3326 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3327
3328 if ((val & zap) != val)
3329 {
3330 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3331 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3332 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3333 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3334 }
3335
3336 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3337 AssertRCReturn(rc, rc);
3338
3339 /* Update VCPU with the currently set VM-exit controls. */
3340 pVCpu->hm.s.vmx.u32EntryCtls = val;
3341 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3342 }
3343 return rc;
3344}
3345
3346
3347/**
3348 * Sets up the VM-exit controls in the VMCS.
3349 *
3350 * @returns VBox status code.
3351 * @param pVM Pointer to the VM.
3352 * @param pVCpu Pointer to the VMCPU.
3353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3354 * out-of-sync. Make sure to update the required fields
3355 * before using them.
3356 *
3357 * @remarks Requires EFER.
3358 */
3359DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3360{
3361 NOREF(pMixedCtx);
3362
3363 int rc = VINF_SUCCESS;
3364 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3365 {
3366 PVM pVM = pVCpu->CTX_SUFF(pVM);
3367 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3368 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3369
3370 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3371 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3372
3373 /*
3374 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3375 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3376 */
3377#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3378 if (HMVMX_IS_64BIT_HOST_MODE())
3379 {
3380 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3381 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3382 }
3383 else
3384 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3385#else
3386 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3387 {
3388 /* The switcher returns to long mode, EFER is managed by the switcher. */
3389 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3390 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3391 }
3392 else
3393 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3394#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3395
3396 /* If the newer VMCS fields for managing EFER exists, use it. */
3397 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3398 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3399 {
3400 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3401 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3402 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3403 }
3404
3405 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3406 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3407
3408 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3409 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3410 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3411
3412 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3413 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3414
3415 if ((val & zap) != val)
3416 {
3417 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3418 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3419 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3420 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3421 }
3422
3423 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3424 AssertRCReturn(rc, rc);
3425
3426 /* Update VCPU with the currently set VM-exit controls. */
3427 pVCpu->hm.s.vmx.u32ExitCtls = val;
3428 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3429 }
3430 return rc;
3431}
3432
3433
3434/**
3435 * Loads the guest APIC and related state.
3436 *
3437 * @returns VBox status code.
3438 * @param pVM Pointer to the VM.
3439 * @param pVCpu Pointer to the VMCPU.
3440 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3441 * out-of-sync. Make sure to update the required fields
3442 * before using them.
3443 */
3444DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3445{
3446 NOREF(pMixedCtx);
3447
3448 int rc = VINF_SUCCESS;
3449 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3450 {
3451 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3452 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3453 {
3454 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3455
3456 bool fPendingIntr = false;
3457 uint8_t u8Tpr = 0;
3458 uint8_t u8PendingIntr = 0;
3459 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3460 AssertRCReturn(rc, rc);
3461
3462 /*
3463 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3464 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3465 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3466 * the interrupt when we VM-exit for other reasons.
3467 */
3468 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3469 uint32_t u32TprThreshold = 0;
3470 if (fPendingIntr)
3471 {
3472 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3473 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3474 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3475 if (u8PendingPriority <= u8TprPriority)
3476 u32TprThreshold = u8PendingPriority;
3477 else
3478 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3479 }
3480 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3481
3482 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3483 AssertRCReturn(rc, rc);
3484 }
3485
3486 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3487 }
3488 return rc;
3489}
3490
3491
3492/**
3493 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3494 *
3495 * @returns Guest's interruptibility-state.
3496 * @param pVCpu Pointer to the VMCPU.
3497 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3498 * out-of-sync. Make sure to update the required fields
3499 * before using them.
3500 *
3501 * @remarks No-long-jump zone!!!
3502 */
3503DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3504{
3505 /*
3506 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3507 * inhibit interrupts or clear any existing interrupt-inhibition.
3508 */
3509 uint32_t uIntrState = 0;
3510 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3511 {
3512 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3513 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3514 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3515 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3516 {
3517 if (pMixedCtx->eflags.Bits.u1IF)
3518 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3519 else
3520 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3521 }
3522 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3523 }
3524
3525 /*
3526 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3527 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3528 * setting this would block host-NMIs and IRET will not clear the blocking.
3529 *
3530 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3531 */
3532 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3533 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3534 {
3535 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3536 }
3537
3538 return uIntrState;
3539}
3540
3541
3542/**
3543 * Loads the guest's interruptibility-state into the guest-state area in the
3544 * VMCS.
3545 *
3546 * @returns VBox status code.
3547 * @param pVCpu Pointer to the VMCPU.
3548 * @param uIntrState The interruptibility-state to set.
3549 */
3550static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3551{
3552 NOREF(pVCpu);
3553 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3554 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3555 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3556 AssertRCReturn(rc, rc);
3557 return rc;
3558}
3559
3560
3561/**
3562 * Loads the guest's RIP into the guest-state area in the VMCS.
3563 *
3564 * @returns VBox status code.
3565 * @param pVCpu Pointer to the VMCPU.
3566 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3567 * out-of-sync. Make sure to update the required fields
3568 * before using them.
3569 *
3570 * @remarks No-long-jump zone!!!
3571 */
3572static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3573{
3574 int rc = VINF_SUCCESS;
3575 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3576 {
3577 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3578 AssertRCReturn(rc, rc);
3579
3580 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3581 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3582 HMCPU_CF_VALUE(pVCpu)));
3583 }
3584 return rc;
3585}
3586
3587
3588/**
3589 * Loads the guest's RSP into the guest-state area in the VMCS.
3590 *
3591 * @returns VBox status code.
3592 * @param pVCpu Pointer to the VMCPU.
3593 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3594 * out-of-sync. Make sure to update the required fields
3595 * before using them.
3596 *
3597 * @remarks No-long-jump zone!!!
3598 */
3599static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3600{
3601 int rc = VINF_SUCCESS;
3602 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3603 {
3604 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3605 AssertRCReturn(rc, rc);
3606
3607 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3608 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3609 }
3610 return rc;
3611}
3612
3613
3614/**
3615 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3616 *
3617 * @returns VBox status code.
3618 * @param pVCpu Pointer to the VMCPU.
3619 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3620 * out-of-sync. Make sure to update the required fields
3621 * before using them.
3622 *
3623 * @remarks No-long-jump zone!!!
3624 */
3625static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3626{
3627 int rc = VINF_SUCCESS;
3628 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3629 {
3630 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3631 Let us assert it as such and use 32-bit VMWRITE. */
3632 Assert(!(pMixedCtx->rflags.u64 >> 32));
3633 X86EFLAGS Eflags = pMixedCtx->eflags;
3634 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3635 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3636
3637 /*
3638 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3639 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3640 */
3641 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3642 {
3643 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3644 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3645 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3646 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3647 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3648 }
3649
3650 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3651 AssertRCReturn(rc, rc);
3652
3653 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3654 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3655 }
3656 return rc;
3657}
3658
3659
3660/**
3661 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3662 *
3663 * @returns VBox status code.
3664 * @param pVCpu Pointer to the VMCPU.
3665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3666 * out-of-sync. Make sure to update the required fields
3667 * before using them.
3668 *
3669 * @remarks No-long-jump zone!!!
3670 */
3671DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3672{
3673 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3674 AssertRCReturn(rc, rc);
3675 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3676 AssertRCReturn(rc, rc);
3677 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3678 AssertRCReturn(rc, rc);
3679 return rc;
3680}
3681
3682
3683/**
3684 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3685 * CR0 is partially shared with the host and we have to consider the FPU bits.
3686 *
3687 * @returns VBox status code.
3688 * @param pVM Pointer to the VM.
3689 * @param pVCpu Pointer to the VMCPU.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775 else
3776 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3777
3778 if (fInterceptNM)
3779 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3780 else
3781 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3782
3783 if (fInterceptMF)
3784 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3785 else
3786 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3787
3788 /* Additional intercepts for debugging, define these yourself explicitly. */
3789#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3790 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3791 | RT_BIT(X86_XCPT_BP)
3792 | RT_BIT(X86_XCPT_DB)
3793 | RT_BIT(X86_XCPT_DE)
3794 | RT_BIT(X86_XCPT_NM)
3795 | RT_BIT(X86_XCPT_TS)
3796 | RT_BIT(X86_XCPT_UD)
3797 | RT_BIT(X86_XCPT_NP)
3798 | RT_BIT(X86_XCPT_SS)
3799 | RT_BIT(X86_XCPT_GP)
3800 | RT_BIT(X86_XCPT_PF)
3801 | RT_BIT(X86_XCPT_MF)
3802 ;
3803#elif defined(HMVMX_ALWAYS_TRAP_PF)
3804 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3805#endif
3806
3807 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3808
3809 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3810 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3811 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3812 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3813 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3814 else
3815 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3816
3817 u32GuestCR0 |= uSetCR0;
3818 u32GuestCR0 &= uZapCR0;
3819 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3820
3821 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3822 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3823 AssertRCReturn(rc, rc);
3824 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3825 AssertRCReturn(rc, rc);
3826 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3827 uZapCR0));
3828
3829 /*
3830 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3831 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3832 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3833 */
3834 uint32_t u32CR0Mask = 0;
3835 u32CR0Mask = X86_CR0_PE
3836 | X86_CR0_NE
3837 | X86_CR0_WP
3838 | X86_CR0_PG
3839 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3840 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3841 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3842
3843 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3844 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3845 * and @bugref{6944}. */
3846#if 0
3847 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3848 u32CR0Mask &= ~X86_CR0_PE;
3849#endif
3850 if (pVM->hm.s.fNestedPaging)
3851 u32CR0Mask &= ~X86_CR0_WP;
3852
3853 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3854 if (fInterceptNM)
3855 {
3856 u32CR0Mask |= X86_CR0_TS
3857 | X86_CR0_MP;
3858 }
3859
3860 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3861 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3862 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3863 AssertRCReturn(rc, rc);
3864 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3865
3866 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3867 }
3868 return rc;
3869}
3870
3871
3872/**
3873 * Loads the guest control registers (CR3, CR4) into the guest-state area
3874 * in the VMCS.
3875 *
3876 * @returns VBox status code.
3877 * @param pVM Pointer to the VM.
3878 * @param pVCpu Pointer to the VMCPU.
3879 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3880 * out-of-sync. Make sure to update the required fields
3881 * before using them.
3882 *
3883 * @remarks No-long-jump zone!!!
3884 */
3885static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3886{
3887 int rc = VINF_SUCCESS;
3888 PVM pVM = pVCpu->CTX_SUFF(pVM);
3889
3890 /*
3891 * Guest CR2.
3892 * It's always loaded in the assembler code. Nothing to do here.
3893 */
3894
3895 /*
3896 * Guest CR3.
3897 */
3898 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3899 {
3900 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3901 if (pVM->hm.s.fNestedPaging)
3902 {
3903 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3904
3905 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3906 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3907 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3908 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3909
3910 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3911 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3912 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3913
3914 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3915 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3916 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3917 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3918
3919 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3920 AssertRCReturn(rc, rc);
3921 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3922
3923 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3924 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3925 {
3926 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3927 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3928 {
3929 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3930 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3931 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3932 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3933 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3934 }
3935
3936 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3937 have Unrestricted Execution to handle the guest when it's not using paging. */
3938 GCPhysGuestCR3 = pMixedCtx->cr3;
3939 }
3940 else
3941 {
3942 /*
3943 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3944 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3945 * EPT takes care of translating it to host-physical addresses.
3946 */
3947 RTGCPHYS GCPhys;
3948 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3949 Assert(PDMVmmDevHeapIsEnabled(pVM));
3950
3951 /* We obtain it here every time as the guest could have relocated this PCI region. */
3952 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3953 AssertRCReturn(rc, rc);
3954
3955 GCPhysGuestCR3 = GCPhys;
3956 }
3957
3958 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3959 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3960 }
3961 else
3962 {
3963 /* Non-nested paging case, just use the hypervisor's CR3. */
3964 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3965
3966 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3967 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3968 }
3969 AssertRCReturn(rc, rc);
3970
3971 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3972 }
3973
3974 /*
3975 * Guest CR4.
3976 */
3977 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3978 {
3979 Assert(!(pMixedCtx->cr4 >> 32));
3980 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3981
3982 /* The guest's view of its CR4 is unblemished. */
3983 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3984 AssertRCReturn(rc, rc);
3985 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3986
3987 /* Setup VT-x's view of the guest CR4. */
3988 /*
3989 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3990 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3991 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3992 */
3993 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3994 {
3995 Assert(pVM->hm.s.vmx.pRealModeTSS);
3996 Assert(PDMVmmDevHeapIsEnabled(pVM));
3997 u32GuestCR4 &= ~X86_CR4_VME;
3998 }
3999
4000 if (pVM->hm.s.fNestedPaging)
4001 {
4002 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4003 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4004 {
4005 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4006 u32GuestCR4 |= X86_CR4_PSE;
4007 /* Our identity mapping is a 32-bit page directory. */
4008 u32GuestCR4 &= ~X86_CR4_PAE;
4009 }
4010 /* else use guest CR4.*/
4011 }
4012 else
4013 {
4014 /*
4015 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4016 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4017 */
4018 switch (pVCpu->hm.s.enmShadowMode)
4019 {
4020 case PGMMODE_REAL: /* Real-mode. */
4021 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4022 case PGMMODE_32_BIT: /* 32-bit paging. */
4023 {
4024 u32GuestCR4 &= ~X86_CR4_PAE;
4025 break;
4026 }
4027
4028 case PGMMODE_PAE: /* PAE paging. */
4029 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4030 {
4031 u32GuestCR4 |= X86_CR4_PAE;
4032 break;
4033 }
4034
4035 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4036 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4037#ifdef VBOX_ENABLE_64_BITS_GUESTS
4038 break;
4039#endif
4040 default:
4041 AssertFailed();
4042 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4043 }
4044 }
4045
4046 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4047 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4048 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4049 u32GuestCR4 |= uSetCR4;
4050 u32GuestCR4 &= uZapCR4;
4051
4052 /* Write VT-x's view of the guest CR4 into the VMCS. */
4053 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4054 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4055 AssertRCReturn(rc, rc);
4056
4057 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4058 uint32_t u32CR4Mask = 0;
4059 u32CR4Mask = X86_CR4_VME
4060 | X86_CR4_PAE
4061 | X86_CR4_PGE
4062 | X86_CR4_PSE
4063 | X86_CR4_VMXE;
4064 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4065 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4066 AssertRCReturn(rc, rc);
4067
4068 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4069 }
4070 return rc;
4071}
4072
4073
4074/**
4075 * Loads the guest debug registers into the guest-state area in the VMCS.
4076 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4077 *
4078 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4079 *
4080 * @returns VBox status code.
4081 * @param pVCpu Pointer to the VMCPU.
4082 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4083 * out-of-sync. Make sure to update the required fields
4084 * before using them.
4085 *
4086 * @remarks No-long-jump zone!!!
4087 */
4088static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4089{
4090 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4091 return VINF_SUCCESS;
4092
4093#ifdef VBOX_STRICT
4094 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4095 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4096 {
4097 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4098 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4099 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4100 }
4101#endif
4102
4103 int rc;
4104 PVM pVM = pVCpu->CTX_SUFF(pVM);
4105 bool fInterceptDB = false;
4106 bool fInterceptMovDRx = false;
4107 if ( pVCpu->hm.s.fSingleInstruction
4108 || DBGFIsStepping(pVCpu))
4109 {
4110 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4111 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4112 {
4113 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4114 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4115 AssertRCReturn(rc, rc);
4116 Assert(fInterceptDB == false);
4117 }
4118 else
4119 {
4120 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4121 pVCpu->hm.s.fClearTrapFlag = true;
4122 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4123 fInterceptDB = true;
4124 }
4125 }
4126
4127 if ( fInterceptDB
4128 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4129 {
4130 /*
4131 * Use the combined guest and host DRx values found in the hypervisor
4132 * register set because the debugger has breakpoints active or someone
4133 * is single stepping on the host side without a monitor trap flag.
4134 *
4135 * Note! DBGF expects a clean DR6 state before executing guest code.
4136 */
4137#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4138 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4139 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4140 {
4141 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4142 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4143 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4144 }
4145 else
4146#endif
4147 if (!CPUMIsHyperDebugStateActive(pVCpu))
4148 {
4149 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4150 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4151 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4152 }
4153
4154 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4155 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4156 AssertRCReturn(rc, rc);
4157
4158 pVCpu->hm.s.fUsingHyperDR7 = true;
4159 fInterceptDB = true;
4160 fInterceptMovDRx = true;
4161 }
4162 else
4163 {
4164 /*
4165 * If the guest has enabled debug registers, we need to load them prior to
4166 * executing guest code so they'll trigger at the right time.
4167 */
4168 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4169 {
4170#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4171 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4172 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4173 {
4174 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4175 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4176 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4177 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4178 }
4179 else
4180#endif
4181 if (!CPUMIsGuestDebugStateActive(pVCpu))
4182 {
4183 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4184 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4185 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4186 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4187 }
4188 Assert(!fInterceptDB);
4189 Assert(!fInterceptMovDRx);
4190 }
4191 /*
4192 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4193 * must intercept #DB in order to maintain a correct DR6 guest value.
4194 */
4195#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4196 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4197 && !CPUMIsGuestDebugStateActive(pVCpu))
4198#else
4199 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4200#endif
4201 {
4202 fInterceptMovDRx = true;
4203 fInterceptDB = true;
4204 }
4205
4206 /* Update guest DR7. */
4207 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4208 AssertRCReturn(rc, rc);
4209
4210 pVCpu->hm.s.fUsingHyperDR7 = false;
4211 }
4212
4213 /*
4214 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4215 */
4216 if (fInterceptDB)
4217 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4218 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4219 {
4220#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4221 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4222#endif
4223 }
4224 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4225 AssertRCReturn(rc, rc);
4226
4227 /*
4228 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4229 */
4230 if (fInterceptMovDRx)
4231 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4232 else
4233 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4234 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4235 AssertRCReturn(rc, rc);
4236
4237 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4238 return VINF_SUCCESS;
4239}
4240
4241
4242#ifdef VBOX_STRICT
4243/**
4244 * Strict function to validate segment registers.
4245 *
4246 * @remarks ASSUMES CR0 is up to date.
4247 */
4248static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4249{
4250 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4251 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4252 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4253 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4254 && ( !CPUMIsGuestInRealModeEx(pCtx)
4255 && !CPUMIsGuestInV86ModeEx(pCtx)))
4256 {
4257 /* Protected mode checks */
4258 /* CS */
4259 Assert(pCtx->cs.Attr.n.u1Present);
4260 Assert(!(pCtx->cs.Attr.u & 0xf00));
4261 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4262 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4263 || !(pCtx->cs.Attr.n.u1Granularity));
4264 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4265 || (pCtx->cs.Attr.n.u1Granularity));
4266 /* CS cannot be loaded with NULL in protected mode. */
4267 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4268 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4269 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4270 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4271 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4272 else
4273 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4274 /* SS */
4275 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4276 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4277 if ( !(pCtx->cr0 & X86_CR0_PE)
4278 || pCtx->cs.Attr.n.u4Type == 3)
4279 {
4280 Assert(!pCtx->ss.Attr.n.u2Dpl);
4281 }
4282 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4283 {
4284 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4285 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4286 Assert(pCtx->ss.Attr.n.u1Present);
4287 Assert(!(pCtx->ss.Attr.u & 0xf00));
4288 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4289 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4290 || !(pCtx->ss.Attr.n.u1Granularity));
4291 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4292 || (pCtx->ss.Attr.n.u1Granularity));
4293 }
4294 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4295 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4296 {
4297 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4298 Assert(pCtx->ds.Attr.n.u1Present);
4299 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4300 Assert(!(pCtx->ds.Attr.u & 0xf00));
4301 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4302 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4303 || !(pCtx->ds.Attr.n.u1Granularity));
4304 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4305 || (pCtx->ds.Attr.n.u1Granularity));
4306 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4307 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4308 }
4309 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4310 {
4311 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4312 Assert(pCtx->es.Attr.n.u1Present);
4313 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4314 Assert(!(pCtx->es.Attr.u & 0xf00));
4315 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4316 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4317 || !(pCtx->es.Attr.n.u1Granularity));
4318 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4319 || (pCtx->es.Attr.n.u1Granularity));
4320 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4321 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4322 }
4323 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4324 {
4325 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4326 Assert(pCtx->fs.Attr.n.u1Present);
4327 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4328 Assert(!(pCtx->fs.Attr.u & 0xf00));
4329 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4330 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4331 || !(pCtx->fs.Attr.n.u1Granularity));
4332 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4333 || (pCtx->fs.Attr.n.u1Granularity));
4334 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4335 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4336 }
4337 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4338 {
4339 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4340 Assert(pCtx->gs.Attr.n.u1Present);
4341 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4342 Assert(!(pCtx->gs.Attr.u & 0xf00));
4343 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4344 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4345 || !(pCtx->gs.Attr.n.u1Granularity));
4346 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4347 || (pCtx->gs.Attr.n.u1Granularity));
4348 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4349 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4350 }
4351 /* 64-bit capable CPUs. */
4352# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4353 if (HMVMX_IS_64BIT_HOST_MODE())
4354 {
4355 Assert(!(pCtx->cs.u64Base >> 32));
4356 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4357 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4358 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4359 }
4360# endif
4361 }
4362 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4363 || ( CPUMIsGuestInRealModeEx(pCtx)
4364 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4365 {
4366 /* Real and v86 mode checks. */
4367 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4368 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4369 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4370 {
4371 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4372 }
4373 else
4374 {
4375 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4376 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4377 }
4378
4379 /* CS */
4380 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4381 Assert(pCtx->cs.u32Limit == 0xffff);
4382 Assert(u32CSAttr == 0xf3);
4383 /* SS */
4384 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4385 Assert(pCtx->ss.u32Limit == 0xffff);
4386 Assert(u32SSAttr == 0xf3);
4387 /* DS */
4388 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4389 Assert(pCtx->ds.u32Limit == 0xffff);
4390 Assert(u32DSAttr == 0xf3);
4391 /* ES */
4392 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4393 Assert(pCtx->es.u32Limit == 0xffff);
4394 Assert(u32ESAttr == 0xf3);
4395 /* FS */
4396 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4397 Assert(pCtx->fs.u32Limit == 0xffff);
4398 Assert(u32FSAttr == 0xf3);
4399 /* GS */
4400 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4401 Assert(pCtx->gs.u32Limit == 0xffff);
4402 Assert(u32GSAttr == 0xf3);
4403 /* 64-bit capable CPUs. */
4404# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4405 if (HMVMX_IS_64BIT_HOST_MODE())
4406 {
4407 Assert(!(pCtx->cs.u64Base >> 32));
4408 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4409 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4410 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4411 }
4412# endif
4413 }
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Writes a guest segment register into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu Pointer to the VMCPU.
4423 * @param idxSel Index of the selector in the VMCS.
4424 * @param idxLimit Index of the segment limit in the VMCS.
4425 * @param idxBase Index of the segment base in the VMCS.
4426 * @param idxAccess Index of the access rights of the segment in the VMCS.
4427 * @param pSelReg Pointer to the segment selector.
4428 *
4429 * @remarks No-long-jump zone!!!
4430 */
4431static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4432 uint32_t idxAccess, PCPUMSELREG pSelReg)
4433{
4434 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4435 AssertRCReturn(rc, rc);
4436 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4437 AssertRCReturn(rc, rc);
4438 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4439 AssertRCReturn(rc, rc);
4440
4441 uint32_t u32Access = pSelReg->Attr.u;
4442 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4443 {
4444 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4445 u32Access = 0xf3;
4446 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4447 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4448 }
4449 else
4450 {
4451 /*
4452 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4453 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4454 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4455 * loaded in protected-mode have their attribute as 0.
4456 */
4457 if (!u32Access)
4458 u32Access = X86DESCATTR_UNUSABLE;
4459 }
4460
4461 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4462 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4463 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4464
4465 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4466 AssertRCReturn(rc, rc);
4467 return rc;
4468}
4469
4470
4471/**
4472 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4473 * into the guest-state area in the VMCS.
4474 *
4475 * @returns VBox status code.
4476 * @param pVM Pointer to the VM.
4477 * @param pVCPU Pointer to the VMCPU.
4478 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4479 * out-of-sync. Make sure to update the required fields
4480 * before using them.
4481 *
4482 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4483 * @remarks No-long-jump zone!!!
4484 */
4485static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4486{
4487 int rc = VERR_INTERNAL_ERROR_5;
4488 PVM pVM = pVCpu->CTX_SUFF(pVM);
4489
4490 /*
4491 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4492 */
4493 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4494 {
4495 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4496 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4497 {
4498 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4501 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4502 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4503 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4504 }
4505
4506#ifdef VBOX_WITH_REM
4507 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4508 {
4509 Assert(pVM->hm.s.vmx.pRealModeTSS);
4510 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4511 if ( pVCpu->hm.s.vmx.fWasInRealMode
4512 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4513 {
4514 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4515 in real-mode (e.g. OpenBSD 4.0) */
4516 REMFlushTBs(pVM);
4517 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4518 pVCpu->hm.s.vmx.fWasInRealMode = false;
4519 }
4520 }
4521#endif
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4523 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4526 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4529 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4532 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4535 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4536 AssertRCReturn(rc, rc);
4537 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4538 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4539 AssertRCReturn(rc, rc);
4540
4541#ifdef VBOX_STRICT
4542 /* Validate. */
4543 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4544#endif
4545
4546 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4547 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4548 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4549 }
4550
4551 /*
4552 * Guest TR.
4553 */
4554 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4555 {
4556 /*
4557 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4558 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4559 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4560 */
4561 uint16_t u16Sel = 0;
4562 uint32_t u32Limit = 0;
4563 uint64_t u64Base = 0;
4564 uint32_t u32AccessRights = 0;
4565
4566 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4567 {
4568 u16Sel = pMixedCtx->tr.Sel;
4569 u32Limit = pMixedCtx->tr.u32Limit;
4570 u64Base = pMixedCtx->tr.u64Base;
4571 u32AccessRights = pMixedCtx->tr.Attr.u;
4572 }
4573 else
4574 {
4575 Assert(pVM->hm.s.vmx.pRealModeTSS);
4576 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4577
4578 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4579 RTGCPHYS GCPhys;
4580 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4581 AssertRCReturn(rc, rc);
4582
4583 X86DESCATTR DescAttr;
4584 DescAttr.u = 0;
4585 DescAttr.n.u1Present = 1;
4586 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4587
4588 u16Sel = 0;
4589 u32Limit = HM_VTX_TSS_SIZE;
4590 u64Base = GCPhys; /* in real-mode phys = virt. */
4591 u32AccessRights = DescAttr.u;
4592 }
4593
4594 /* Validate. */
4595 Assert(!(u16Sel & RT_BIT(2)));
4596 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4597 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4598 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4599 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4600 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4601 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4602 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4603 Assert( (u32Limit & 0xfff) == 0xfff
4604 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4605 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4606 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4607
4608 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4609 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4610 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4611 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4612
4613 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4614 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4615 }
4616
4617 /*
4618 * Guest GDTR.
4619 */
4620 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4621 {
4622 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4623 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4624
4625 /* Validate. */
4626 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4627
4628 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4629 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4630 }
4631
4632 /*
4633 * Guest LDTR.
4634 */
4635 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4636 {
4637 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4638 uint32_t u32Access = 0;
4639 if (!pMixedCtx->ldtr.Attr.u)
4640 u32Access = X86DESCATTR_UNUSABLE;
4641 else
4642 u32Access = pMixedCtx->ldtr.Attr.u;
4643
4644 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4645 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4646 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4647 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4661 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4672 {
4673 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4674 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4675
4676 /* Validate. */
4677 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4678
4679 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4680 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4681 }
4682
4683 return VINF_SUCCESS;
4684}
4685
4686
4687/**
4688 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4689 * areas. These MSRs will automatically be loaded to the host CPU on every
4690 * successful VM entry and stored from the host CPU on every successful VM-exit.
4691 *
4692 * This also creates/updates MSR slots for the host MSRs. The actual host
4693 * MSR values are -not- updated here for performance reasons. See
4694 * hmR0VmxSaveHostMsrs().
4695 *
4696 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4697 *
4698 * @returns VBox status code.
4699 * @param pVCpu Pointer to the VMCPU.
4700 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4701 * out-of-sync. Make sure to update the required fields
4702 * before using them.
4703 *
4704 * @remarks No-long-jump zone!!!
4705 */
4706static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4707{
4708 AssertPtr(pVCpu);
4709 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4710
4711 /*
4712 * MSRs that we use the auto-load/store MSR area in the VMCS.
4713 */
4714 PVM pVM = pVCpu->CTX_SUFF(pVM);
4715 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4716 {
4717 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4718#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4719 if (pVM->hm.s.fAllow64BitGuests)
4720 {
4721 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4722 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4723 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4724 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4725# ifdef DEBUG
4726 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4727 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4728 {
4729 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4730 pMsr->u64Value));
4731 }
4732# endif
4733 }
4734#endif
4735 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4736 }
4737
4738 /*
4739 * Guest Sysenter MSRs.
4740 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4741 * VM-exits on WRMSRs for these MSRs.
4742 */
4743 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4744 {
4745 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4746 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4747 }
4748
4749 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4750 {
4751 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4752 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4753 }
4754
4755 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4756 {
4757 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4758 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4759 }
4760
4761 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4762 {
4763 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4764 {
4765 /*
4766 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4767 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4768 */
4769 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4770 {
4771 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4772 AssertRCReturn(rc,rc);
4773 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4774 }
4775 else
4776 {
4777 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4778 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4779 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4780 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4781 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4782 }
4783 }
4784 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4785 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4786 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4787 }
4788
4789 return VINF_SUCCESS;
4790}
4791
4792
4793/**
4794 * Loads the guest activity state into the guest-state area in the VMCS.
4795 *
4796 * @returns VBox status code.
4797 * @param pVCpu Pointer to the VMCPU.
4798 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4799 * out-of-sync. Make sure to update the required fields
4800 * before using them.
4801 *
4802 * @remarks No-long-jump zone!!!
4803 */
4804static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4805{
4806 NOREF(pCtx);
4807 /** @todo See if we can make use of other states, e.g.
4808 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4809 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4810 {
4811 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4812 AssertRCReturn(rc, rc);
4813
4814 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4815 }
4816 return VINF_SUCCESS;
4817}
4818
4819
4820/**
4821 * Sets up the appropriate function to run guest code.
4822 *
4823 * @returns VBox status code.
4824 * @param pVCpu Pointer to the VMCPU.
4825 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4826 * out-of-sync. Make sure to update the required fields
4827 * before using them.
4828 *
4829 * @remarks No-long-jump zone!!!
4830 */
4831static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4832{
4833 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4834 {
4835#ifndef VBOX_ENABLE_64_BITS_GUESTS
4836 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4837#endif
4838 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4839#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4840 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4841 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4842 {
4843 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4844 {
4845 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4846 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4847 | HM_CHANGED_VMX_EXIT_CTLS
4848 | HM_CHANGED_VMX_ENTRY_CTLS
4849 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4850 }
4851 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4852 }
4853#else
4854 /* 64-bit host or hybrid host. */
4855 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4856#endif
4857 }
4858 else
4859 {
4860 /* Guest is not in long mode, use the 32-bit handler. */
4861#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4862 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4863 {
4864 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4865 {
4866 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4867 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4868 | HM_CHANGED_VMX_EXIT_CTLS
4869 | HM_CHANGED_VMX_ENTRY_CTLS
4870 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4871 }
4872 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4873 }
4874#else
4875 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4876#endif
4877 }
4878 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4879 return VINF_SUCCESS;
4880}
4881
4882
4883/**
4884 * Wrapper for running the guest code in VT-x.
4885 *
4886 * @returns VBox strict status code.
4887 * @param pVM Pointer to the VM.
4888 * @param pVCpu Pointer to the VMCPU.
4889 * @param pCtx Pointer to the guest-CPU context.
4890 *
4891 * @remarks No-long-jump zone!!!
4892 */
4893DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4894{
4895 /*
4896 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4897 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4898 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4899 */
4900 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4901 /** @todo Add stats for resume vs launch. */
4902#ifdef VBOX_WITH_KERNEL_USING_XMM
4903 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4904#else
4905 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4906#endif
4907}
4908
4909
4910/**
4911 * Reports world-switch error and dumps some useful debug info.
4912 *
4913 * @param pVM Pointer to the VM.
4914 * @param pVCpu Pointer to the VMCPU.
4915 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4916 * @param pCtx Pointer to the guest-CPU context.
4917 * @param pVmxTransient Pointer to the VMX transient structure (only
4918 * exitReason updated).
4919 */
4920static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4921{
4922 Assert(pVM);
4923 Assert(pVCpu);
4924 Assert(pCtx);
4925 Assert(pVmxTransient);
4926 HMVMX_ASSERT_PREEMPT_SAFE();
4927
4928 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4929 switch (rcVMRun)
4930 {
4931 case VERR_VMX_INVALID_VMXON_PTR:
4932 AssertFailed();
4933 break;
4934 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4935 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4936 {
4937 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4938 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4939 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4940 AssertRC(rc);
4941
4942 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4943 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4944 Cannot do it here as we may have been long preempted. */
4945
4946#ifdef VBOX_STRICT
4947 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4948 pVmxTransient->uExitReason));
4949 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4950 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4951 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4952 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4953 else
4954 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4955 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4956 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4957
4958 /* VMX control bits. */
4959 uint32_t u32Val;
4960 uint64_t u64Val;
4961 HMVMXHCUINTREG uHCReg;
4962 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4963 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4964 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4965 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4992 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4993 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4994 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4995 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4997 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4999 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5000 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5001 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5002 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5003 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5004
5005 /* Guest bits. */
5006 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5007 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5008 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5009 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5010 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5011 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5012 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5013 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5014
5015 /* Host bits. */
5016 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5017 Log4(("Host CR0 %#RHr\n", uHCReg));
5018 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5019 Log4(("Host CR3 %#RHr\n", uHCReg));
5020 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5021 Log4(("Host CR4 %#RHr\n", uHCReg));
5022
5023 RTGDTR HostGdtr;
5024 PCX86DESCHC pDesc;
5025 ASMGetGDTR(&HostGdtr);
5026 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5027 Log4(("Host CS %#08x\n", u32Val));
5028 if (u32Val < HostGdtr.cbGdt)
5029 {
5030 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5031 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5032 }
5033
5034 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5035 Log4(("Host DS %#08x\n", u32Val));
5036 if (u32Val < HostGdtr.cbGdt)
5037 {
5038 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5039 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5040 }
5041
5042 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5043 Log4(("Host ES %#08x\n", u32Val));
5044 if (u32Val < HostGdtr.cbGdt)
5045 {
5046 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5047 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5048 }
5049
5050 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5051 Log4(("Host FS %#08x\n", u32Val));
5052 if (u32Val < HostGdtr.cbGdt)
5053 {
5054 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5055 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5056 }
5057
5058 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5059 Log4(("Host GS %#08x\n", u32Val));
5060 if (u32Val < HostGdtr.cbGdt)
5061 {
5062 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5063 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5064 }
5065
5066 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5067 Log4(("Host SS %#08x\n", u32Val));
5068 if (u32Val < HostGdtr.cbGdt)
5069 {
5070 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5071 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5072 }
5073
5074 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5075 Log4(("Host TR %#08x\n", u32Val));
5076 if (u32Val < HostGdtr.cbGdt)
5077 {
5078 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5079 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5080 }
5081
5082 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5083 Log4(("Host TR Base %#RHv\n", uHCReg));
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5085 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5087 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5088 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5089 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5091 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5093 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5095 Log4(("Host RSP %#RHv\n", uHCReg));
5096 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5097 Log4(("Host RIP %#RHv\n", uHCReg));
5098# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5099 if (HMVMX_IS_64BIT_HOST_MODE())
5100 {
5101 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5102 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5103 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5104 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5105 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5106 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5107 }
5108# endif
5109#endif /* VBOX_STRICT */
5110 break;
5111 }
5112
5113 default:
5114 /* Impossible */
5115 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5116 break;
5117 }
5118 NOREF(pVM); NOREF(pCtx);
5119}
5120
5121
5122#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5123#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5124# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5125#endif
5126#ifdef VBOX_STRICT
5127static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5128{
5129 switch (idxField)
5130 {
5131 case VMX_VMCS_GUEST_RIP:
5132 case VMX_VMCS_GUEST_RSP:
5133 case VMX_VMCS_GUEST_SYSENTER_EIP:
5134 case VMX_VMCS_GUEST_SYSENTER_ESP:
5135 case VMX_VMCS_GUEST_GDTR_BASE:
5136 case VMX_VMCS_GUEST_IDTR_BASE:
5137 case VMX_VMCS_GUEST_CS_BASE:
5138 case VMX_VMCS_GUEST_DS_BASE:
5139 case VMX_VMCS_GUEST_ES_BASE:
5140 case VMX_VMCS_GUEST_FS_BASE:
5141 case VMX_VMCS_GUEST_GS_BASE:
5142 case VMX_VMCS_GUEST_SS_BASE:
5143 case VMX_VMCS_GUEST_LDTR_BASE:
5144 case VMX_VMCS_GUEST_TR_BASE:
5145 case VMX_VMCS_GUEST_CR3:
5146 return true;
5147 }
5148 return false;
5149}
5150
5151static bool hmR0VmxIsValidReadField(uint32_t idxField)
5152{
5153 switch (idxField)
5154 {
5155 /* Read-only fields. */
5156 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5157 return true;
5158 }
5159 /* Remaining readable fields should also be writable. */
5160 return hmR0VmxIsValidWriteField(idxField);
5161}
5162#endif /* VBOX_STRICT */
5163
5164
5165/**
5166 * Executes the specified handler in 64-bit mode.
5167 *
5168 * @returns VBox status code.
5169 * @param pVM Pointer to the VM.
5170 * @param pVCpu Pointer to the VMCPU.
5171 * @param pCtx Pointer to the guest CPU context.
5172 * @param enmOp The operation to perform.
5173 * @param cbParam Number of parameters.
5174 * @param paParam Array of 32-bit parameters.
5175 */
5176VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5177 uint32_t *paParam)
5178{
5179 int rc, rc2;
5180 PHMGLOBALCPUINFO pCpu;
5181 RTHCPHYS HCPhysCpuPage;
5182 RTCCUINTREG uOldEflags;
5183
5184 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5185 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5186 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5187 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5188
5189#ifdef VBOX_STRICT
5190 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5191 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5192
5193 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5194 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5195#endif
5196
5197 /* Disable interrupts. */
5198 uOldEflags = ASMIntDisableFlags();
5199
5200#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5201 RTCPUID idHostCpu = RTMpCpuId();
5202 CPUMR0SetLApic(pVCpu, idHostCpu);
5203#endif
5204
5205 pCpu = HMR0GetCurrentCpu();
5206 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5207
5208 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5209 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5210
5211 /* Leave VMX Root Mode. */
5212 VMXDisable();
5213
5214 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5215
5216 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5217 CPUMSetHyperEIP(pVCpu, enmOp);
5218 for (int i = (int)cbParam - 1; i >= 0; i--)
5219 CPUMPushHyper(pVCpu, paParam[i]);
5220
5221 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5222
5223 /* Call the switcher. */
5224 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5225 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5226
5227 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5228 /* Make sure the VMX instructions don't cause #UD faults. */
5229 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5230
5231 /* Re-enter VMX Root Mode */
5232 rc2 = VMXEnable(HCPhysCpuPage);
5233 if (RT_FAILURE(rc2))
5234 {
5235 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5236 ASMSetFlags(uOldEflags);
5237 return rc2;
5238 }
5239
5240 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5241 AssertRC(rc2);
5242 Assert(!(ASMGetFlags() & X86_EFL_IF));
5243 ASMSetFlags(uOldEflags);
5244 return rc;
5245}
5246
5247
5248/**
5249 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5250 * supporting 64-bit guests.
5251 *
5252 * @returns VBox status code.
5253 * @param fResume Whether to VMLAUNCH or VMRESUME.
5254 * @param pCtx Pointer to the guest-CPU context.
5255 * @param pCache Pointer to the VMCS cache.
5256 * @param pVM Pointer to the VM.
5257 * @param pVCpu Pointer to the VMCPU.
5258 */
5259DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5260{
5261 uint32_t aParam[6];
5262 PHMGLOBALCPUINFO pCpu = NULL;
5263 RTHCPHYS HCPhysCpuPage = 0;
5264 int rc = VERR_INTERNAL_ERROR_5;
5265
5266 pCpu = HMR0GetCurrentCpu();
5267 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5268
5269#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5270 pCache->uPos = 1;
5271 pCache->interPD = PGMGetInterPaeCR3(pVM);
5272 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5273#endif
5274
5275#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5276 pCache->TestIn.HCPhysCpuPage = 0;
5277 pCache->TestIn.HCPhysVmcs = 0;
5278 pCache->TestIn.pCache = 0;
5279 pCache->TestOut.HCPhysVmcs = 0;
5280 pCache->TestOut.pCache = 0;
5281 pCache->TestOut.pCtx = 0;
5282 pCache->TestOut.eflags = 0;
5283#endif
5284
5285 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5286 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5287 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5288 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5289 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5290 aParam[5] = 0;
5291
5292#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5293 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5294 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5295#endif
5296 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5297
5298#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5299 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5300 Assert(pCtx->dr[4] == 10);
5301 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5302#endif
5303
5304#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5305 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5306 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5307 pVCpu->hm.s.vmx.HCPhysVmcs));
5308 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5309 pCache->TestOut.HCPhysVmcs));
5310 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5311 pCache->TestOut.pCache));
5312 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5313 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5314 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5315 pCache->TestOut.pCtx));
5316 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5317#endif
5318 return rc;
5319}
5320
5321
5322/**
5323 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5324 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5325 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5326 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5327 *
5328 * @returns VBox status code.
5329 * @param pVM Pointer to the VM.
5330 * @param pVCpu Pointer to the VMCPU.
5331 */
5332static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5333{
5334#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5335{ \
5336 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5337 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5338 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5339 ++cReadFields; \
5340}
5341
5342 AssertPtr(pVM);
5343 AssertPtr(pVCpu);
5344 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5345 uint32_t cReadFields = 0;
5346
5347 /*
5348 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5349 * and serve to indicate exceptions to the rules.
5350 */
5351
5352 /* Guest-natural selector base fields. */
5353#if 0
5354 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5357#endif
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5370#if 0
5371 /* Unused natural width guest-state fields. */
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5374#endif
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5377
5378 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5379#if 0
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5389#endif
5390
5391 /* Natural width guest-state fields. */
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5393#if 0
5394 /* Currently unused field. */
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5396#endif
5397
5398 if (pVM->hm.s.fNestedPaging)
5399 {
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5401 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5402 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5403 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5404 }
5405 else
5406 {
5407 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5408 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5409 }
5410
5411#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5412 return VINF_SUCCESS;
5413}
5414
5415
5416/**
5417 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5418 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5419 * darwin, running 64-bit guests).
5420 *
5421 * @returns VBox status code.
5422 * @param pVCpu Pointer to the VMCPU.
5423 * @param idxField The VMCS field encoding.
5424 * @param u64Val 16, 32 or 64-bit value.
5425 */
5426VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5427{
5428 int rc;
5429 switch (idxField)
5430 {
5431 /*
5432 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5433 */
5434 /* 64-bit Control fields. */
5435 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5436 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5437 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5438 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5439 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5440 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5441 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5442 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5443 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5444 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5445 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5446 case VMX_VMCS64_CTRL_EPTP_FULL:
5447 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5448 /* 64-bit Guest-state fields. */
5449 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5450 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5451 case VMX_VMCS64_GUEST_PAT_FULL:
5452 case VMX_VMCS64_GUEST_EFER_FULL:
5453 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5454 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5455 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5456 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5457 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5458 /* 64-bit Host-state fields. */
5459 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5460 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5461 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5462 {
5463 rc = VMXWriteVmcs32(idxField, u64Val);
5464 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5465 break;
5466 }
5467
5468 /*
5469 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5470 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5471 */
5472 /* Natural-width Guest-state fields. */
5473 case VMX_VMCS_GUEST_CR3:
5474 case VMX_VMCS_GUEST_ES_BASE:
5475 case VMX_VMCS_GUEST_CS_BASE:
5476 case VMX_VMCS_GUEST_SS_BASE:
5477 case VMX_VMCS_GUEST_DS_BASE:
5478 case VMX_VMCS_GUEST_FS_BASE:
5479 case VMX_VMCS_GUEST_GS_BASE:
5480 case VMX_VMCS_GUEST_LDTR_BASE:
5481 case VMX_VMCS_GUEST_TR_BASE:
5482 case VMX_VMCS_GUEST_GDTR_BASE:
5483 case VMX_VMCS_GUEST_IDTR_BASE:
5484 case VMX_VMCS_GUEST_RSP:
5485 case VMX_VMCS_GUEST_RIP:
5486 case VMX_VMCS_GUEST_SYSENTER_ESP:
5487 case VMX_VMCS_GUEST_SYSENTER_EIP:
5488 {
5489 if (!(u64Val >> 32))
5490 {
5491 /* If this field is 64-bit, VT-x will zero out the top bits. */
5492 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5493 }
5494 else
5495 {
5496 /* Assert that only the 32->64 switcher case should ever come here. */
5497 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5498 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5499 }
5500 break;
5501 }
5502
5503 default:
5504 {
5505 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5506 rc = VERR_INVALID_PARAMETER;
5507 break;
5508 }
5509 }
5510 AssertRCReturn(rc, rc);
5511 return rc;
5512}
5513
5514
5515/**
5516 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5517 * hosts (except darwin) for 64-bit guests.
5518 *
5519 * @param pVCpu Pointer to the VMCPU.
5520 * @param idxField The VMCS field encoding.
5521 * @param u64Val 16, 32 or 64-bit value.
5522 */
5523VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5524{
5525 AssertPtr(pVCpu);
5526 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5527
5528 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5529 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5530
5531 /* Make sure there are no duplicates. */
5532 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5533 {
5534 if (pCache->Write.aField[i] == idxField)
5535 {
5536 pCache->Write.aFieldVal[i] = u64Val;
5537 return VINF_SUCCESS;
5538 }
5539 }
5540
5541 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5542 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5543 pCache->Write.cValidEntries++;
5544 return VINF_SUCCESS;
5545}
5546
5547/* Enable later when the assembly code uses these as callbacks. */
5548#if 0
5549/*
5550 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5551 *
5552 * @param pVCpu Pointer to the VMCPU.
5553 * @param pCache Pointer to the VMCS cache.
5554 *
5555 * @remarks No-long-jump zone!!!
5556 */
5557VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5558{
5559 AssertPtr(pCache);
5560 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5561 {
5562 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5563 AssertRC(rc);
5564 }
5565 pCache->Write.cValidEntries = 0;
5566}
5567
5568
5569/**
5570 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5571 *
5572 * @param pVCpu Pointer to the VMCPU.
5573 * @param pCache Pointer to the VMCS cache.
5574 *
5575 * @remarks No-long-jump zone!!!
5576 */
5577VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5578{
5579 AssertPtr(pCache);
5580 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5581 {
5582 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5583 AssertRC(rc);
5584 }
5585}
5586#endif
5587#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5588
5589
5590/**
5591 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5592 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5593 * timer.
5594 *
5595 * @returns VBox status code.
5596 * @param pVCpu Pointer to the VMCPU.
5597 *
5598 * @remarks No-long-jump zone!!!
5599 */
5600static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5601{
5602 int rc = VERR_INTERNAL_ERROR_5;
5603 bool fOffsettedTsc = false;
5604 bool fParavirtTsc = false;
5605 PVM pVM = pVCpu->CTX_SUFF(pVM);
5606 if (pVM->hm.s.vmx.fUsePreemptTimer)
5607 {
5608 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5609 &pVCpu->hm.s.vmx.u64TSCOffset);
5610
5611 /* Make sure the returned values have sane upper and lower boundaries. */
5612 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5613 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5614 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5615 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5616
5617 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5618 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5619 }
5620 else
5621 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5622
5623#if 1
5624 if (fParavirtTsc)
5625 {
5626#if 1
5627 uint64_t const u64CurTsc = ASMReadTSC();
5628 uint64_t const u64LastTick = TMCpuTickGetLastSeen(pVCpu);
5629 if (u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset < u64LastTick)
5630 {
5631 pVCpu->hm.s.vmx.u64TSCOffset = (u64LastTick - u64CurTsc);
5632 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffsetAdjusted);
5633 }
5634
5635 Assert(u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset >= u64LastTick);
5636#endif
5637 rc = GIMR0UpdateParavirtTsc(pVM, pVCpu->hm.s.vmx.u64TSCOffset);
5638 AssertRC(rc);
5639 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5640 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRC(rc);
5641
5642 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5643 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5644 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5645 }
5646 else
5647#else
5648 if (fParavirtTsc)
5649 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5650#endif
5651 if (fOffsettedTsc)
5652 {
5653 uint64_t u64CurTSC = ASMReadTSC();
5654 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5655 {
5656 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5657 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5658
5659 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5660 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5661 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5662 }
5663 else
5664 {
5665 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5666 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5667 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5668 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5669 }
5670 }
5671 else
5672 {
5673 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5674 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5675 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5676 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5677 }
5678}
5679
5680
5681/**
5682 * Determines if an exception is a contributory exception. Contributory
5683 * exceptions are ones which can cause double-faults. Page-fault is
5684 * intentionally not included here as it's a conditional contributory exception.
5685 *
5686 * @returns true if the exception is contributory, false otherwise.
5687 * @param uVector The exception vector.
5688 */
5689DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5690{
5691 switch (uVector)
5692 {
5693 case X86_XCPT_GP:
5694 case X86_XCPT_SS:
5695 case X86_XCPT_NP:
5696 case X86_XCPT_TS:
5697 case X86_XCPT_DE:
5698 return true;
5699 default:
5700 break;
5701 }
5702 return false;
5703}
5704
5705
5706/**
5707 * Sets an event as a pending event to be injected into the guest.
5708 *
5709 * @param pVCpu Pointer to the VMCPU.
5710 * @param u32IntInfo The VM-entry interruption-information field.
5711 * @param cbInstr The VM-entry instruction length in bytes (for software
5712 * interrupts, exceptions and privileged software
5713 * exceptions).
5714 * @param u32ErrCode The VM-entry exception error code.
5715 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5716 * page-fault.
5717 *
5718 * @remarks Statistics counter assumes this is a guest event being injected or
5719 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5720 * always incremented.
5721 */
5722DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5723 RTGCUINTPTR GCPtrFaultAddress)
5724{
5725 Assert(!pVCpu->hm.s.Event.fPending);
5726 pVCpu->hm.s.Event.fPending = true;
5727 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5728 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5729 pVCpu->hm.s.Event.cbInstr = cbInstr;
5730 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5731
5732 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5733}
5734
5735
5736/**
5737 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5738 *
5739 * @param pVCpu Pointer to the VMCPU.
5740 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5741 * out-of-sync. Make sure to update the required fields
5742 * before using them.
5743 */
5744DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5745{
5746 NOREF(pMixedCtx);
5747 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5748 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5749 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5750 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5751}
5752
5753
5754/**
5755 * Handle a condition that occurred while delivering an event through the guest
5756 * IDT.
5757 *
5758 * @returns VBox status code (informational error codes included).
5759 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5760 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5761 * continue execution of the guest which will delivery the #DF.
5762 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5763 *
5764 * @param pVCpu Pointer to the VMCPU.
5765 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5766 * out-of-sync. Make sure to update the required fields
5767 * before using them.
5768 * @param pVmxTransient Pointer to the VMX transient structure.
5769 *
5770 * @remarks No-long-jump zone!!!
5771 */
5772static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5773{
5774 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5775
5776 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5777 AssertRCReturn(rc, rc);
5778 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5779 {
5780 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5781 AssertRCReturn(rc, rc);
5782
5783 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5784 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5785
5786 typedef enum
5787 {
5788 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5789 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5790 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5791 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5792 } VMXREFLECTXCPT;
5793
5794 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5795 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5796 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5797 {
5798 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5799 {
5800 enmReflect = VMXREFLECTXCPT_XCPT;
5801#ifdef VBOX_STRICT
5802 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5803 && uExitVector == X86_XCPT_PF)
5804 {
5805 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5806 }
5807#endif
5808 if ( uExitVector == X86_XCPT_PF
5809 && uIdtVector == X86_XCPT_PF)
5810 {
5811 pVmxTransient->fVectoringPF = true;
5812 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5813 }
5814 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5815 && hmR0VmxIsContributoryXcpt(uExitVector)
5816 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5817 || uIdtVector == X86_XCPT_PF))
5818 {
5819 enmReflect = VMXREFLECTXCPT_DF;
5820 }
5821 else if (uIdtVector == X86_XCPT_DF)
5822 enmReflect = VMXREFLECTXCPT_TF;
5823 }
5824 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5825 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5826 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5827 {
5828 /*
5829 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5830 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5831 */
5832 enmReflect = VMXREFLECTXCPT_XCPT;
5833 }
5834 }
5835 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5836 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5837 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5838 {
5839 /*
5840 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5841 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5842 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5843 */
5844 enmReflect = VMXREFLECTXCPT_XCPT;
5845 }
5846
5847 /*
5848 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5849 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5850 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5851 *
5852 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5853 */
5854 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5855 && enmReflect == VMXREFLECTXCPT_XCPT
5856 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5857 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5858 {
5859 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5860 }
5861
5862 switch (enmReflect)
5863 {
5864 case VMXREFLECTXCPT_XCPT:
5865 {
5866 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5867 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5868 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5869
5870 uint32_t u32ErrCode = 0;
5871 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5872 {
5873 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5874 AssertRCReturn(rc, rc);
5875 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5876 }
5877
5878 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5879 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5880 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5881 rc = VINF_SUCCESS;
5882 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5883 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5884
5885 break;
5886 }
5887
5888 case VMXREFLECTXCPT_DF:
5889 {
5890 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5891 rc = VINF_HM_DOUBLE_FAULT;
5892 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5893 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5894
5895 break;
5896 }
5897
5898 case VMXREFLECTXCPT_TF:
5899 {
5900 rc = VINF_EM_RESET;
5901 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5902 uExitVector));
5903 break;
5904 }
5905
5906 default:
5907 Assert(rc == VINF_SUCCESS);
5908 break;
5909 }
5910 }
5911 else if ( VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5912 && uExitVector != X86_XCPT_DF
5913 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5914 {
5915 /*
5916 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5917 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5918 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5919 */
5920 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5921 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5922 }
5923
5924 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5925 return rc;
5926}
5927
5928
5929/**
5930 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5931 *
5932 * @returns VBox status code.
5933 * @param pVCpu Pointer to the VMCPU.
5934 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5935 * out-of-sync. Make sure to update the required fields
5936 * before using them.
5937 *
5938 * @remarks No-long-jump zone!!!
5939 */
5940static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5941{
5942 NOREF(pMixedCtx);
5943
5944 /*
5945 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5946 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5947 */
5948 VMMRZCallRing3Disable(pVCpu);
5949 HM_DISABLE_PREEMPT_IF_NEEDED();
5950
5951 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5952 {
5953 uint32_t uVal = 0;
5954 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5955 AssertRCReturn(rc, rc);
5956
5957 uint32_t uShadow = 0;
5958 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5959 AssertRCReturn(rc, rc);
5960
5961 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5962 CPUMSetGuestCR0(pVCpu, uVal);
5963 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5964 }
5965
5966 HM_RESTORE_PREEMPT_IF_NEEDED();
5967 VMMRZCallRing3Enable(pVCpu);
5968 return VINF_SUCCESS;
5969}
5970
5971
5972/**
5973 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5974 *
5975 * @returns VBox status code.
5976 * @param pVCpu Pointer to the VMCPU.
5977 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5978 * out-of-sync. Make sure to update the required fields
5979 * before using them.
5980 *
5981 * @remarks No-long-jump zone!!!
5982 */
5983static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5984{
5985 NOREF(pMixedCtx);
5986
5987 int rc = VINF_SUCCESS;
5988 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5989 {
5990 uint32_t uVal = 0;
5991 uint32_t uShadow = 0;
5992 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5993 AssertRCReturn(rc, rc);
5994 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5995 AssertRCReturn(rc, rc);
5996
5997 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5998 CPUMSetGuestCR4(pVCpu, uVal);
5999 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6000 }
6001 return rc;
6002}
6003
6004
6005/**
6006 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6007 *
6008 * @returns VBox status code.
6009 * @param pVCpu Pointer to the VMCPU.
6010 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6011 * out-of-sync. Make sure to update the required fields
6012 * before using them.
6013 *
6014 * @remarks No-long-jump zone!!!
6015 */
6016static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6017{
6018 int rc = VINF_SUCCESS;
6019 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6020 {
6021 uint64_t u64Val = 0;
6022 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6023 AssertRCReturn(rc, rc);
6024
6025 pMixedCtx->rip = u64Val;
6026 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6027 }
6028 return rc;
6029}
6030
6031
6032/**
6033 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6034 *
6035 * @returns VBox status code.
6036 * @param pVCpu Pointer to the VMCPU.
6037 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6038 * out-of-sync. Make sure to update the required fields
6039 * before using them.
6040 *
6041 * @remarks No-long-jump zone!!!
6042 */
6043static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6044{
6045 int rc = VINF_SUCCESS;
6046 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6047 {
6048 uint64_t u64Val = 0;
6049 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6050 AssertRCReturn(rc, rc);
6051
6052 pMixedCtx->rsp = u64Val;
6053 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6054 }
6055 return rc;
6056}
6057
6058
6059/**
6060 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6061 *
6062 * @returns VBox status code.
6063 * @param pVCpu Pointer to the VMCPU.
6064 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6065 * out-of-sync. Make sure to update the required fields
6066 * before using them.
6067 *
6068 * @remarks No-long-jump zone!!!
6069 */
6070static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6071{
6072 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6073 {
6074 uint32_t uVal = 0;
6075 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6076 AssertRCReturn(rc, rc);
6077
6078 pMixedCtx->eflags.u32 = uVal;
6079 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6080 {
6081 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6082 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6083
6084 pMixedCtx->eflags.Bits.u1VM = 0;
6085 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6086 }
6087
6088 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6089 }
6090 return VINF_SUCCESS;
6091}
6092
6093
6094/**
6095 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6096 * guest-CPU context.
6097 */
6098DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6099{
6100 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6101 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6102 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6103 return rc;
6104}
6105
6106
6107/**
6108 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6109 * from the guest-state area in the VMCS.
6110 *
6111 * @param pVCpu Pointer to the VMCPU.
6112 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6113 * out-of-sync. Make sure to update the required fields
6114 * before using them.
6115 *
6116 * @remarks No-long-jump zone!!!
6117 */
6118static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6119{
6120 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6121 {
6122 uint32_t uIntrState = 0;
6123 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6124 AssertRC(rc);
6125
6126 if (!uIntrState)
6127 {
6128 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6129 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6130
6131 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6132 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6133 }
6134 else
6135 {
6136 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6137 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6138 {
6139 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6140 AssertRC(rc);
6141 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6142 AssertRC(rc);
6143
6144 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6145 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6146 }
6147
6148 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6149 {
6150 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6151 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6152 }
6153 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6154 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6155 }
6156
6157 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6158 }
6159}
6160
6161
6162/**
6163 * Saves the guest's activity state.
6164 *
6165 * @returns VBox status code.
6166 * @param pVCpu Pointer to the VMCPU.
6167 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6168 * out-of-sync. Make sure to update the required fields
6169 * before using them.
6170 *
6171 * @remarks No-long-jump zone!!!
6172 */
6173static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6174{
6175 NOREF(pMixedCtx);
6176 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6177 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6178 return VINF_SUCCESS;
6179}
6180
6181
6182/**
6183 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6184 * the current VMCS into the guest-CPU context.
6185 *
6186 * @returns VBox status code.
6187 * @param pVCpu Pointer to the VMCPU.
6188 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6189 * out-of-sync. Make sure to update the required fields
6190 * before using them.
6191 *
6192 * @remarks No-long-jump zone!!!
6193 */
6194static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6195{
6196 int rc = VINF_SUCCESS;
6197 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6198 {
6199 uint32_t u32Val = 0;
6200 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6201 pMixedCtx->SysEnter.cs = u32Val;
6202 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6203 }
6204
6205 uint64_t u64Val = 0;
6206 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6207 {
6208 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6209 pMixedCtx->SysEnter.eip = u64Val;
6210 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6211 }
6212 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6213 {
6214 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6215 pMixedCtx->SysEnter.esp = u64Val;
6216 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6217 }
6218 return rc;
6219}
6220
6221
6222/**
6223 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6224 * the CPU back into the guest-CPU context.
6225 *
6226 * @returns VBox status code.
6227 * @param pVCpu Pointer to the VMCPU.
6228 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6229 * out-of-sync. Make sure to update the required fields
6230 * before using them.
6231 *
6232 * @remarks No-long-jump zone!!!
6233 */
6234static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6235{
6236#if HC_ARCH_BITS == 64
6237 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6238 {
6239 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6240 VMMRZCallRing3Disable(pVCpu);
6241 HM_DISABLE_PREEMPT_IF_NEEDED();
6242
6243 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6244 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6245 {
6246 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6247 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6248 }
6249
6250 HM_RESTORE_PREEMPT_IF_NEEDED();
6251 VMMRZCallRing3Enable(pVCpu);
6252 }
6253 else
6254 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6255#else
6256 NOREF(pMixedCtx);
6257 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6258#endif
6259
6260 return VINF_SUCCESS;
6261}
6262
6263
6264/**
6265 * Saves the auto load/store'd guest MSRs from the current VMCS into
6266 * the guest-CPU context.
6267 *
6268 * @returns VBox status code.
6269 * @param pVCpu Pointer to the VMCPU.
6270 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6271 * out-of-sync. Make sure to update the required fields
6272 * before using them.
6273 *
6274 * @remarks No-long-jump zone!!!
6275 */
6276static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6277{
6278 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6279 return VINF_SUCCESS;
6280
6281 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6282 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6283 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6284 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6285 {
6286 switch (pMsr->u32Msr)
6287 {
6288 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6289 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6290 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6291 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6292 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6293 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6294 break;
6295
6296 default:
6297 {
6298 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6299 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6300 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6301 }
6302 }
6303 }
6304
6305 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6306 return VINF_SUCCESS;
6307}
6308
6309
6310/**
6311 * Saves the guest control registers from the current VMCS into the guest-CPU
6312 * context.
6313 *
6314 * @returns VBox status code.
6315 * @param pVCpu Pointer to the VMCPU.
6316 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6317 * out-of-sync. Make sure to update the required fields
6318 * before using them.
6319 *
6320 * @remarks No-long-jump zone!!!
6321 */
6322static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6323{
6324 /* Guest CR0. Guest FPU. */
6325 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6326 AssertRCReturn(rc, rc);
6327
6328 /* Guest CR4. */
6329 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6330 AssertRCReturn(rc, rc);
6331
6332 /* Guest CR2 - updated always during the world-switch or in #PF. */
6333 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6334 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6335 {
6336 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6337 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6338
6339 PVM pVM = pVCpu->CTX_SUFF(pVM);
6340 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6341 || ( pVM->hm.s.fNestedPaging
6342 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6343 {
6344 uint64_t u64Val = 0;
6345 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6346 if (pMixedCtx->cr3 != u64Val)
6347 {
6348 CPUMSetGuestCR3(pVCpu, u64Val);
6349 if (VMMRZCallRing3IsEnabled(pVCpu))
6350 {
6351 PGMUpdateCR3(pVCpu, u64Val);
6352 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6353 }
6354 else
6355 {
6356 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6357 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6358 }
6359 }
6360
6361 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6362 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6363 {
6364 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6365 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6366 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6367 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6368
6369 if (VMMRZCallRing3IsEnabled(pVCpu))
6370 {
6371 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6372 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6373 }
6374 else
6375 {
6376 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6377 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6378 }
6379 }
6380 }
6381
6382 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6383 }
6384
6385 /*
6386 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6387 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6388 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6389 *
6390 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6391 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6392 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6393 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6394 *
6395 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6396 */
6397 if (VMMRZCallRing3IsEnabled(pVCpu))
6398 {
6399 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6400 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6401
6402 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6403 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6404
6405 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6406 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6407 }
6408
6409 return rc;
6410}
6411
6412
6413/**
6414 * Reads a guest segment register from the current VMCS into the guest-CPU
6415 * context.
6416 *
6417 * @returns VBox status code.
6418 * @param pVCpu Pointer to the VMCPU.
6419 * @param idxSel Index of the selector in the VMCS.
6420 * @param idxLimit Index of the segment limit in the VMCS.
6421 * @param idxBase Index of the segment base in the VMCS.
6422 * @param idxAccess Index of the access rights of the segment in the VMCS.
6423 * @param pSelReg Pointer to the segment selector.
6424 *
6425 * @remarks No-long-jump zone!!!
6426 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6427 * macro as that takes care of whether to read from the VMCS cache or
6428 * not.
6429 */
6430DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6431 PCPUMSELREG pSelReg)
6432{
6433 NOREF(pVCpu);
6434
6435 uint32_t u32Val = 0;
6436 int rc = VMXReadVmcs32(idxSel, &u32Val);
6437 AssertRCReturn(rc, rc);
6438 pSelReg->Sel = (uint16_t)u32Val;
6439 pSelReg->ValidSel = (uint16_t)u32Val;
6440 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6441
6442 rc = VMXReadVmcs32(idxLimit, &u32Val);
6443 AssertRCReturn(rc, rc);
6444 pSelReg->u32Limit = u32Val;
6445
6446 uint64_t u64Val = 0;
6447 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6448 AssertRCReturn(rc, rc);
6449 pSelReg->u64Base = u64Val;
6450
6451 rc = VMXReadVmcs32(idxAccess, &u32Val);
6452 AssertRCReturn(rc, rc);
6453 pSelReg->Attr.u = u32Val;
6454
6455 /*
6456 * If VT-x marks the segment as unusable, most other bits remain undefined:
6457 * - For CS the L, D and G bits have meaning.
6458 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6459 * - For the remaining data segments no bits are defined.
6460 *
6461 * The present bit and the unusable bit has been observed to be set at the
6462 * same time (the selector was supposed to be invalid as we started executing
6463 * a V8086 interrupt in ring-0).
6464 *
6465 * What should be important for the rest of the VBox code, is that the P bit is
6466 * cleared. Some of the other VBox code recognizes the unusable bit, but
6467 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6468 * safe side here, we'll strip off P and other bits we don't care about. If
6469 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6470 *
6471 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6472 */
6473 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6474 {
6475 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6476
6477 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6478 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6479 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6480
6481 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6482#ifdef DEBUG_bird
6483 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6484 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6485 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6486#endif
6487 }
6488 return VINF_SUCCESS;
6489}
6490
6491
6492#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6493# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6494 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6495 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6496#else
6497# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6498 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6499 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6500#endif
6501
6502
6503/**
6504 * Saves the guest segment registers from the current VMCS into the guest-CPU
6505 * context.
6506 *
6507 * @returns VBox status code.
6508 * @param pVCpu Pointer to the VMCPU.
6509 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6510 * out-of-sync. Make sure to update the required fields
6511 * before using them.
6512 *
6513 * @remarks No-long-jump zone!!!
6514 */
6515static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6516{
6517 /* Guest segment registers. */
6518 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6519 {
6520 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6521 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6522 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6523 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6524 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6525 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6526 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6527
6528 /* Restore segment attributes for real-on-v86 mode hack. */
6529 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6530 {
6531 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6532 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6533 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6534 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6535 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6536 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6537 }
6538 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6539 }
6540
6541 return VINF_SUCCESS;
6542}
6543
6544
6545/**
6546 * Saves the guest descriptor table registers and task register from the current
6547 * VMCS into the guest-CPU context.
6548 *
6549 * @returns VBox status code.
6550 * @param pVCpu Pointer to the VMCPU.
6551 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6552 * out-of-sync. Make sure to update the required fields
6553 * before using them.
6554 *
6555 * @remarks No-long-jump zone!!!
6556 */
6557static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6558{
6559 int rc = VINF_SUCCESS;
6560
6561 /* Guest LDTR. */
6562 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6563 {
6564 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6565 AssertRCReturn(rc, rc);
6566 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6567 }
6568
6569 /* Guest GDTR. */
6570 uint64_t u64Val = 0;
6571 uint32_t u32Val = 0;
6572 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6573 {
6574 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6575 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6576 pMixedCtx->gdtr.pGdt = u64Val;
6577 pMixedCtx->gdtr.cbGdt = u32Val;
6578 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6579 }
6580
6581 /* Guest IDTR. */
6582 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6583 {
6584 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6585 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6586 pMixedCtx->idtr.pIdt = u64Val;
6587 pMixedCtx->idtr.cbIdt = u32Val;
6588 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6589 }
6590
6591 /* Guest TR. */
6592 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6593 {
6594 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6595 AssertRCReturn(rc, rc);
6596
6597 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6598 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6599 {
6600 rc = VMXLOCAL_READ_SEG(TR, tr);
6601 AssertRCReturn(rc, rc);
6602 }
6603 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6604 }
6605 return rc;
6606}
6607
6608#undef VMXLOCAL_READ_SEG
6609
6610
6611/**
6612 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6613 * context.
6614 *
6615 * @returns VBox status code.
6616 * @param pVCpu Pointer to the VMCPU.
6617 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6618 * out-of-sync. Make sure to update the required fields
6619 * before using them.
6620 *
6621 * @remarks No-long-jump zone!!!
6622 */
6623static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6624{
6625 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6626 {
6627 if (!pVCpu->hm.s.fUsingHyperDR7)
6628 {
6629 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6630 uint32_t u32Val;
6631 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6632 pMixedCtx->dr[7] = u32Val;
6633 }
6634
6635 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6636 }
6637 return VINF_SUCCESS;
6638}
6639
6640
6641/**
6642 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6643 *
6644 * @returns VBox status code.
6645 * @param pVCpu Pointer to the VMCPU.
6646 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6647 * out-of-sync. Make sure to update the required fields
6648 * before using them.
6649 *
6650 * @remarks No-long-jump zone!!!
6651 */
6652static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6653{
6654 NOREF(pMixedCtx);
6655
6656 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6657 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6658 return VINF_SUCCESS;
6659}
6660
6661
6662/**
6663 * Saves the entire guest state from the currently active VMCS into the
6664 * guest-CPU context. This essentially VMREADs all guest-data.
6665 *
6666 * @returns VBox status code.
6667 * @param pVCpu Pointer to the VMCPU.
6668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6669 * out-of-sync. Make sure to update the required fields
6670 * before using them.
6671 */
6672static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6673{
6674 Assert(pVCpu);
6675 Assert(pMixedCtx);
6676
6677 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6678 return VINF_SUCCESS;
6679
6680 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6681 again on the ring-3 callback path, there is no real need to. */
6682 if (VMMRZCallRing3IsEnabled(pVCpu))
6683 VMMR0LogFlushDisable(pVCpu);
6684 else
6685 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6686 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6687
6688 /* In case we get preempted before saving the interruptibility-state in hmR0VmxPostRunGuest(), do it here.
6689 Otherwise we lose the info. from the VMCS if we get rescheduled on a different host CPU. */
6690 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6691
6692 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6693 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6694
6695 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6696 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6697
6698 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6699 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6700
6701 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6702 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6703
6704 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6705 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6706
6707 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6708 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6709
6710 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6711 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6712
6713 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6714 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6715
6716 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6717 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6718
6719 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6720 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6721
6722 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6723 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6724
6725 if (VMMRZCallRing3IsEnabled(pVCpu))
6726 VMMR0LogFlushEnable(pVCpu);
6727
6728 return rc;
6729}
6730
6731
6732/**
6733 * Check per-VM and per-VCPU force flag actions that require us to go back to
6734 * ring-3 for one reason or another.
6735 *
6736 * @returns VBox status code (information status code included).
6737 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6738 * ring-3.
6739 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6740 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6741 * interrupts)
6742 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6743 * all EMTs to be in ring-3.
6744 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6745 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6746 * to the EM loop.
6747 *
6748 * @param pVM Pointer to the VM.
6749 * @param pVCpu Pointer to the VMCPU.
6750 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6751 * out-of-sync. Make sure to update the required fields
6752 * before using them.
6753 */
6754static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6755{
6756 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6757
6758 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6759 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6760 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6761 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6762 {
6763 /* We need the control registers now, make sure the guest-CPU context is updated. */
6764 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6765 AssertRCReturn(rc3, rc3);
6766
6767 /* Pending HM CR3 sync. */
6768 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6769 {
6770 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6771 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6772 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6773 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6774 }
6775
6776 /* Pending HM PAE PDPEs. */
6777 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6778 {
6779 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6780 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6781 }
6782
6783 /* Pending PGM C3 sync. */
6784 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6785 {
6786 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6787 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6788 if (rc2 != VINF_SUCCESS)
6789 {
6790 AssertRC(rc2);
6791 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6792 return rc2;
6793 }
6794 }
6795
6796 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6797 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6798 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6799 {
6800 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6801 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6802 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6803 return rc2;
6804 }
6805
6806 /* Pending VM request packets, such as hardware interrupts. */
6807 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6808 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6809 {
6810 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6811 return VINF_EM_PENDING_REQUEST;
6812 }
6813
6814 /* Pending PGM pool flushes. */
6815 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6816 {
6817 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6818 return VINF_PGM_POOL_FLUSH_PENDING;
6819 }
6820
6821 /* Pending DMA requests. */
6822 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6823 {
6824 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6825 return VINF_EM_RAW_TO_R3;
6826 }
6827 }
6828
6829 return VINF_SUCCESS;
6830}
6831
6832
6833/**
6834 * Converts any TRPM trap into a pending HM event. This is typically used when
6835 * entering from ring-3 (not longjmp returns).
6836 *
6837 * @param pVCpu Pointer to the VMCPU.
6838 */
6839static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6840{
6841 Assert(TRPMHasTrap(pVCpu));
6842 Assert(!pVCpu->hm.s.Event.fPending);
6843
6844 uint8_t uVector;
6845 TRPMEVENT enmTrpmEvent;
6846 RTGCUINT uErrCode;
6847 RTGCUINTPTR GCPtrFaultAddress;
6848 uint8_t cbInstr;
6849
6850 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6851 AssertRC(rc);
6852
6853 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6854 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6855 if (enmTrpmEvent == TRPM_TRAP)
6856 {
6857 switch (uVector)
6858 {
6859 case X86_XCPT_NMI:
6860 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6861 break;
6862
6863 case X86_XCPT_BP:
6864 case X86_XCPT_OF:
6865 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6866 break;
6867
6868 case X86_XCPT_PF:
6869 case X86_XCPT_DF:
6870 case X86_XCPT_TS:
6871 case X86_XCPT_NP:
6872 case X86_XCPT_SS:
6873 case X86_XCPT_GP:
6874 case X86_XCPT_AC:
6875 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6876 /* no break! */
6877 default:
6878 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6879 break;
6880 }
6881 }
6882 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6883 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6884 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6885 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6886 else
6887 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6888
6889 rc = TRPMResetTrap(pVCpu);
6890 AssertRC(rc);
6891 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6892 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6893
6894 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6895 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6896}
6897
6898
6899/**
6900 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6901 * VT-x to execute any instruction.
6902 *
6903 * @param pvCpu Pointer to the VMCPU.
6904 */
6905static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6906{
6907 Assert(pVCpu->hm.s.Event.fPending);
6908
6909 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6910 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6911 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6912 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6913
6914 /* If a trap was already pending, we did something wrong! */
6915 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6916
6917 TRPMEVENT enmTrapType;
6918 switch (uVectorType)
6919 {
6920 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6921 enmTrapType = TRPM_HARDWARE_INT;
6922 break;
6923
6924 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6925 enmTrapType = TRPM_SOFTWARE_INT;
6926 break;
6927
6928 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6929 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6930 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6931 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6932 enmTrapType = TRPM_TRAP;
6933 break;
6934
6935 default:
6936 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6937 enmTrapType = TRPM_32BIT_HACK;
6938 break;
6939 }
6940
6941 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6942
6943 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6944 AssertRC(rc);
6945
6946 if (fErrorCodeValid)
6947 TRPMSetErrorCode(pVCpu, uErrorCode);
6948
6949 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6950 && uVector == X86_XCPT_PF)
6951 {
6952 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6953 }
6954 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6955 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6956 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6957 {
6958 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6959 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6960 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6961 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6962 }
6963 pVCpu->hm.s.Event.fPending = false;
6964}
6965
6966
6967/**
6968 * Does the necessary state syncing before returning to ring-3 for any reason
6969 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6970 *
6971 * @returns VBox status code.
6972 * @param pVM Pointer to the VM.
6973 * @param pVCpu Pointer to the VMCPU.
6974 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6975 * be out-of-sync. Make sure to update the required
6976 * fields before using them.
6977 * @param fSaveGuestState Whether to save the guest state or not.
6978 *
6979 * @remarks No-long-jmp zone!!!
6980 */
6981static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6982{
6983 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6984 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6985
6986 RTCPUID idCpu = RTMpCpuId();
6987 Log4Func(("HostCpuId=%u\n", idCpu));
6988
6989 /*
6990 * !!! IMPORTANT !!!
6991 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6992 */
6993
6994 /* Save the guest state if necessary. */
6995 if ( fSaveGuestState
6996 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6997 {
6998 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6999 AssertRCReturn(rc, rc);
7000 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7001 }
7002
7003 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7004 if (CPUMIsGuestFPUStateActive(pVCpu))
7005 {
7006 /* We shouldn't reload CR0 without saving it first. */
7007 if (!fSaveGuestState)
7008 {
7009 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7010 AssertRCReturn(rc, rc);
7011 }
7012 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7013 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7014 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7015 }
7016
7017 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7018#ifdef VBOX_STRICT
7019 if (CPUMIsHyperDebugStateActive(pVCpu))
7020 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7021#endif
7022 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7023 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7024 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7025 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7026
7027#if HC_ARCH_BITS == 64
7028 /* Restore host-state bits that VT-x only restores partially. */
7029 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7030 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7031 {
7032 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7033 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7034 }
7035 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7036#endif
7037
7038#if HC_ARCH_BITS == 64
7039 /* Restore the host MSRs as we're leaving VT-x context. */
7040 if ( pVM->hm.s.fAllow64BitGuests
7041 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7042 {
7043 /* We shouldn't reload the guest MSRs without saving it first. */
7044 if (!fSaveGuestState)
7045 {
7046 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7047 AssertRCReturn(rc, rc);
7048 }
7049 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7050 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7051 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
7052 }
7053#endif
7054
7055 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7056 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7057
7058 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7059 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7060 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7061 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7062 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7063 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7064 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7065 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7066
7067 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7068
7069 /** @todo This partially defeats the purpose of having preemption hooks.
7070 * The problem is, deregistering the hooks should be moved to a place that
7071 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7072 * context.
7073 */
7074 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7075 {
7076 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7077 AssertRCReturn(rc, rc);
7078
7079 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7080 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7081 }
7082 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7083 NOREF(idCpu);
7084
7085 return VINF_SUCCESS;
7086}
7087
7088
7089/**
7090 * Leaves the VT-x session.
7091 *
7092 * @returns VBox status code.
7093 * @param pVM Pointer to the VM.
7094 * @param pVCpu Pointer to the VMCPU.
7095 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7096 * out-of-sync. Make sure to update the required fields
7097 * before using them.
7098 *
7099 * @remarks No-long-jmp zone!!!
7100 */
7101DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7102{
7103 HM_DISABLE_PREEMPT_IF_NEEDED();
7104 HMVMX_ASSERT_CPU_SAFE();
7105 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7106 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7107
7108 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7109 and done this from the VMXR0ThreadCtxCallback(). */
7110 if (!pVCpu->hm.s.fLeaveDone)
7111 {
7112 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7113 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7114 pVCpu->hm.s.fLeaveDone = true;
7115 }
7116 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7117
7118 /*
7119 * !!! IMPORTANT !!!
7120 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7121 */
7122
7123 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7124 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7125 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7126 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7127 VMMR0ThreadCtxHooksDeregister(pVCpu);
7128
7129 /* Leave HM context. This takes care of local init (term). */
7130 int rc = HMR0LeaveCpu(pVCpu);
7131
7132 HM_RESTORE_PREEMPT_IF_NEEDED();
7133
7134 return rc;
7135}
7136
7137
7138/**
7139 * Does the necessary state syncing before doing a longjmp to ring-3.
7140 *
7141 * @returns VBox status code.
7142 * @param pVM Pointer to the VM.
7143 * @param pVCpu Pointer to the VMCPU.
7144 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7145 * out-of-sync. Make sure to update the required fields
7146 * before using them.
7147 *
7148 * @remarks No-long-jmp zone!!!
7149 */
7150DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7151{
7152 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7153}
7154
7155
7156/**
7157 * Take necessary actions before going back to ring-3.
7158 *
7159 * An action requires us to go back to ring-3. This function does the necessary
7160 * steps before we can safely return to ring-3. This is not the same as longjmps
7161 * to ring-3, this is voluntary and prepares the guest so it may continue
7162 * executing outside HM (recompiler/IEM).
7163 *
7164 * @returns VBox status code.
7165 * @param pVM Pointer to the VM.
7166 * @param pVCpu Pointer to the VMCPU.
7167 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7168 * out-of-sync. Make sure to update the required fields
7169 * before using them.
7170 * @param rcExit The reason for exiting to ring-3. Can be
7171 * VINF_VMM_UNKNOWN_RING3_CALL.
7172 */
7173static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7174{
7175 Assert(pVM);
7176 Assert(pVCpu);
7177 Assert(pMixedCtx);
7178 HMVMX_ASSERT_PREEMPT_SAFE();
7179
7180 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7181 {
7182 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7183 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7184 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7185 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7186 }
7187
7188 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7189 VMMRZCallRing3Disable(pVCpu);
7190 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7191
7192 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7193 if (pVCpu->hm.s.Event.fPending)
7194 {
7195 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7196 Assert(!pVCpu->hm.s.Event.fPending);
7197 }
7198
7199 /* Save guest state and restore host state bits. */
7200 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7201 AssertRCReturn(rc, rc);
7202 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7203
7204 /* Sync recompiler state. */
7205 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7206 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7207 | CPUM_CHANGED_LDTR
7208 | CPUM_CHANGED_GDTR
7209 | CPUM_CHANGED_IDTR
7210 | CPUM_CHANGED_TR
7211 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7212 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7213 if ( pVM->hm.s.fNestedPaging
7214 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7215 {
7216 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7217 }
7218
7219 Assert(!pVCpu->hm.s.fClearTrapFlag);
7220
7221 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7222 if (rcExit != VINF_EM_RAW_INTERRUPT)
7223 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7224
7225 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7226
7227 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7228 VMMRZCallRing3RemoveNotification(pVCpu);
7229 VMMRZCallRing3Enable(pVCpu);
7230
7231 return rc;
7232}
7233
7234
7235/**
7236 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7237 * longjump to ring-3 and possibly get preempted.
7238 *
7239 * @returns VBox status code.
7240 * @param pVCpu Pointer to the VMCPU.
7241 * @param enmOperation The operation causing the ring-3 longjump.
7242 * @param pvUser Opaque pointer to the guest-CPU context. The data
7243 * may be out-of-sync. Make sure to update the required
7244 * fields before using them.
7245 */
7246DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7247{
7248 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7249 {
7250 /*
7251 * !!! IMPORTANT !!!
7252 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7253 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7254 */
7255 VMMRZCallRing3RemoveNotification(pVCpu);
7256 VMMRZCallRing3Disable(pVCpu);
7257 HM_DISABLE_PREEMPT_IF_NEEDED();
7258
7259 PVM pVM = pVCpu->CTX_SUFF(pVM);
7260 if (CPUMIsGuestFPUStateActive(pVCpu))
7261 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7262
7263 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7264
7265#if HC_ARCH_BITS == 64
7266 /* Restore host-state bits that VT-x only restores partially. */
7267 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7268 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7269 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7270 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7271
7272 /* Restore the host MSRs as we're leaving VT-x context. */
7273 if ( pVM->hm.s.fAllow64BitGuests
7274 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7275 {
7276 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7277 }
7278#endif
7279 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7280 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7281 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7282 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7283 {
7284 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7285 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7286 }
7287
7288 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7289 VMMR0ThreadCtxHooksDeregister(pVCpu);
7290
7291 HMR0LeaveCpu(pVCpu);
7292 HM_RESTORE_PREEMPT_IF_NEEDED();
7293 return VINF_SUCCESS;
7294 }
7295
7296 Assert(pVCpu);
7297 Assert(pvUser);
7298 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7299 HMVMX_ASSERT_PREEMPT_SAFE();
7300
7301 VMMRZCallRing3Disable(pVCpu);
7302 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7303
7304 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7305 enmOperation));
7306
7307 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7308 AssertRCReturn(rc, rc);
7309
7310 VMMRZCallRing3Enable(pVCpu);
7311 return VINF_SUCCESS;
7312}
7313
7314
7315/**
7316 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7317 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7318 *
7319 * @param pVCpu Pointer to the VMCPU.
7320 */
7321DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7322{
7323 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7324 {
7325 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7326 {
7327 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7328 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7329 AssertRC(rc);
7330 Log4(("Setup interrupt-window exiting\n"));
7331 }
7332 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7333}
7334
7335
7336/**
7337 * Clears the interrupt-window exiting control in the VMCS.
7338 *
7339 * @param pVCpu Pointer to the VMCPU.
7340 */
7341DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7342{
7343 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7344 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7345 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7346 AssertRC(rc);
7347 Log4(("Cleared interrupt-window exiting\n"));
7348}
7349
7350
7351/**
7352 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7353 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7354 *
7355 * @param pVCpu Pointer to the VMCPU.
7356 */
7357DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7358{
7359 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7360 {
7361 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7362 {
7363 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7364 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7365 AssertRC(rc);
7366 Log4(("Setup NMI-window exiting\n"));
7367 }
7368 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7369}
7370
7371
7372/**
7373 * Clears the NMI-window exiting control in the VMCS.
7374 *
7375 * @param pVCpu Pointer to the VMCPU.
7376 */
7377DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7378{
7379 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7380 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7381 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7382 AssertRC(rc);
7383 Log4(("Cleared NMI-window exiting\n"));
7384}
7385
7386
7387/**
7388 * Evaluates the event to be delivered to the guest and sets it as the pending
7389 * event.
7390 *
7391 * @param pVCpu Pointer to the VMCPU.
7392 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7393 * out-of-sync. Make sure to update the required fields
7394 * before using them.
7395 */
7396static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7397{
7398 Assert(!pVCpu->hm.s.Event.fPending);
7399
7400 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7401 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7402 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7403 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7404 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7405
7406 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7407 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7408 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7409 Assert(!TRPMHasTrap(pVCpu));
7410
7411 /*
7412 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7413 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7414 */
7415 /** @todo SMI. SMIs take priority over NMIs. */
7416 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7417 {
7418 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7419 if ( !fBlockNmi
7420 && !fBlockSti
7421 && !fBlockMovSS)
7422 {
7423 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7424 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7425 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7426
7427 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7428 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7429 }
7430 else
7431 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7432 }
7433 /*
7434 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7435 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7436 */
7437 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7438 && !pVCpu->hm.s.fSingleInstruction)
7439 {
7440 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7441 AssertRC(rc);
7442 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7443 if ( !fBlockInt
7444 && !fBlockSti
7445 && !fBlockMovSS)
7446 {
7447 uint8_t u8Interrupt;
7448 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7449 if (RT_SUCCESS(rc))
7450 {
7451 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7452 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7453 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7454
7455 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7456 }
7457 else
7458 {
7459 /** @todo Does this actually happen? If not turn it into an assertion. */
7460 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7461 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7462 }
7463 }
7464 else
7465 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7466 }
7467}
7468
7469
7470/**
7471 * Sets a pending-debug exception to be delivered to the guest if the guest is
7472 * single-stepping.
7473 *
7474 * @param pVCpu Pointer to the VMCPU.
7475 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7476 * out-of-sync. Make sure to update the required fields
7477 * before using them.
7478 */
7479DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7480{
7481 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7482 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7483 {
7484 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7485 AssertRC(rc);
7486 }
7487}
7488
7489
7490/**
7491 * Injects any pending events into the guest if the guest is in a state to
7492 * receive them.
7493 *
7494 * @returns VBox status code (informational status codes included).
7495 * @param pVCpu Pointer to the VMCPU.
7496 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7497 * out-of-sync. Make sure to update the required fields
7498 * before using them.
7499 */
7500static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7501{
7502 HMVMX_ASSERT_PREEMPT_SAFE();
7503 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7504
7505 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7506 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7507 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7508 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7509
7510 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7511 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7512 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7513 Assert(!TRPMHasTrap(pVCpu));
7514
7515 int rc = VINF_SUCCESS;
7516 if (pVCpu->hm.s.Event.fPending)
7517 {
7518 /*
7519 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7520 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7521 * ended up enabling interrupts outside VT-x.
7522 */
7523 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7524 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7525 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7526 {
7527 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7528 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7529 }
7530
7531#ifdef VBOX_STRICT
7532 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7533 {
7534 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7535 Assert(!fBlockInt);
7536 Assert(!fBlockSti);
7537 Assert(!fBlockMovSS);
7538 }
7539 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7540 {
7541 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7542 Assert(!fBlockSti);
7543 Assert(!fBlockMovSS);
7544 Assert(!fBlockNmi);
7545 }
7546#endif
7547 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7548 (uint8_t)uIntType));
7549 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7550 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7551 AssertRCReturn(rc, rc);
7552
7553 /* Update the interruptibility-state as it could have been changed by
7554 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7555 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7556 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7557
7558#ifdef VBOX_WITH_STATISTICS
7559 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7560 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7561 else
7562 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7563#endif
7564 }
7565
7566 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7567 if ( fBlockSti
7568 || fBlockMovSS)
7569 {
7570 if ( !pVCpu->hm.s.fSingleInstruction
7571 && !DBGFIsStepping(pVCpu))
7572 {
7573 /*
7574 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7575 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7576 * See Intel spec. 27.3.4 "Saving Non-Register State".
7577 */
7578 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7579 AssertRCReturn(rc2, rc2);
7580 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7581 }
7582 else if (pMixedCtx->eflags.Bits.u1TF)
7583 {
7584 /*
7585 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7586 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7587 */
7588 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7589 uIntrState = 0;
7590 }
7591 }
7592
7593 /*
7594 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7595 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7596 */
7597 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7598 AssertRC(rc2);
7599
7600 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7601 NOREF(fBlockMovSS); NOREF(fBlockSti);
7602 return rc;
7603}
7604
7605
7606/**
7607 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7608 *
7609 * @param pVCpu Pointer to the VMCPU.
7610 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7611 * out-of-sync. Make sure to update the required fields
7612 * before using them.
7613 */
7614DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7615{
7616 NOREF(pMixedCtx);
7617 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7618 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7619}
7620
7621
7622/**
7623 * Injects a double-fault (#DF) exception into the VM.
7624 *
7625 * @returns VBox status code (informational status code included).
7626 * @param pVCpu Pointer to the VMCPU.
7627 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7628 * out-of-sync. Make sure to update the required fields
7629 * before using them.
7630 */
7631DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7632{
7633 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7634 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7635 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7636 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7637 puIntrState);
7638}
7639
7640
7641/**
7642 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7643 *
7644 * @param pVCpu Pointer to the VMCPU.
7645 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7646 * out-of-sync. Make sure to update the required fields
7647 * before using them.
7648 */
7649DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7650{
7651 NOREF(pMixedCtx);
7652 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7653 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7654 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7655}
7656
7657
7658/**
7659 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7660 *
7661 * @param pVCpu Pointer to the VMCPU.
7662 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7663 * out-of-sync. Make sure to update the required fields
7664 * before using them.
7665 * @param cbInstr The value of RIP that is to be pushed on the guest
7666 * stack.
7667 */
7668DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7669{
7670 NOREF(pMixedCtx);
7671 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7672 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7673 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7674}
7675
7676
7677/**
7678 * Injects a general-protection (#GP) fault into the VM.
7679 *
7680 * @returns VBox status code (informational status code included).
7681 * @param pVCpu Pointer to the VMCPU.
7682 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7683 * out-of-sync. Make sure to update the required fields
7684 * before using them.
7685 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7686 * mode, i.e. in real-mode it's not valid).
7687 * @param u32ErrorCode The error code associated with the #GP.
7688 */
7689DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7690 uint32_t *puIntrState)
7691{
7692 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7693 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7694 if (fErrorCodeValid)
7695 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7696 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7697 puIntrState);
7698}
7699
7700
7701/**
7702 * Sets a general-protection (#GP) exception as pending-for-injection into the
7703 * VM.
7704 *
7705 * @param pVCpu Pointer to the VMCPU.
7706 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7707 * out-of-sync. Make sure to update the required fields
7708 * before using them.
7709 * @param u32ErrorCode The error code associated with the #GP.
7710 */
7711DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7712{
7713 NOREF(pMixedCtx);
7714 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7715 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7716 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7717 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7718}
7719
7720
7721/**
7722 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7723 *
7724 * @param pVCpu Pointer to the VMCPU.
7725 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7726 * out-of-sync. Make sure to update the required fields
7727 * before using them.
7728 * @param uVector The software interrupt vector number.
7729 * @param cbInstr The value of RIP that is to be pushed on the guest
7730 * stack.
7731 */
7732DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7733{
7734 NOREF(pMixedCtx);
7735 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7736 if ( uVector == X86_XCPT_BP
7737 || uVector == X86_XCPT_OF)
7738 {
7739 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7740 }
7741 else
7742 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7743 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7744}
7745
7746
7747/**
7748 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7749 * stack.
7750 *
7751 * @returns VBox status code (information status code included).
7752 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7753 * @param pVM Pointer to the VM.
7754 * @param pMixedCtx Pointer to the guest-CPU context.
7755 * @param uValue The value to push to the guest stack.
7756 */
7757DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7758{
7759 /*
7760 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7761 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7762 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7763 */
7764 if (pMixedCtx->sp == 1)
7765 return VINF_EM_RESET;
7766 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7767 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7768 AssertRCReturn(rc, rc);
7769 return rc;
7770}
7771
7772
7773/**
7774 * Injects an event into the guest upon VM-entry by updating the relevant fields
7775 * in the VM-entry area in the VMCS.
7776 *
7777 * @returns VBox status code (informational error codes included).
7778 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7779 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7780 *
7781 * @param pVCpu Pointer to the VMCPU.
7782 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7783 * be out-of-sync. Make sure to update the required
7784 * fields before using them.
7785 * @param u64IntInfo The VM-entry interruption-information field.
7786 * @param cbInstr The VM-entry instruction length in bytes (for
7787 * software interrupts, exceptions and privileged
7788 * software exceptions).
7789 * @param u32ErrCode The VM-entry exception error code.
7790 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7791 * @param puIntrState Pointer to the current guest interruptibility-state.
7792 * This interruptibility-state will be updated if
7793 * necessary. This cannot not be NULL.
7794 *
7795 * @remarks Requires CR0!
7796 * @remarks No-long-jump zone!!!
7797 */
7798static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7799 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7800{
7801 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7802 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7803 Assert(puIntrState);
7804 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7805
7806 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7807 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7808
7809#ifdef VBOX_STRICT
7810 /* Validate the error-code-valid bit for hardware exceptions. */
7811 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7812 {
7813 switch (uVector)
7814 {
7815 case X86_XCPT_PF:
7816 case X86_XCPT_DF:
7817 case X86_XCPT_TS:
7818 case X86_XCPT_NP:
7819 case X86_XCPT_SS:
7820 case X86_XCPT_GP:
7821 case X86_XCPT_AC:
7822 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7823 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7824 /* fallthru */
7825 default:
7826 break;
7827 }
7828 }
7829#endif
7830
7831 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7832 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7833 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7834
7835 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7836
7837 /* We require CR0 to check if the guest is in real-mode. */
7838 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7839 AssertRCReturn(rc, rc);
7840
7841 /*
7842 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7843 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7844 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7845 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7846 */
7847 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7848 {
7849 PVM pVM = pVCpu->CTX_SUFF(pVM);
7850 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7851 {
7852 Assert(PDMVmmDevHeapIsEnabled(pVM));
7853 Assert(pVM->hm.s.vmx.pRealModeTSS);
7854
7855 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7856 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7857 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7858 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7859 AssertRCReturn(rc, rc);
7860 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7861
7862 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7863 size_t const cbIdtEntry = sizeof(X86IDTR16);
7864 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7865 {
7866 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7867 if (uVector == X86_XCPT_DF)
7868 return VINF_EM_RESET;
7869 else if (uVector == X86_XCPT_GP)
7870 {
7871 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7872 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7873 }
7874
7875 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7876 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7877 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7878 }
7879
7880 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7881 uint16_t uGuestIp = pMixedCtx->ip;
7882 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7883 {
7884 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7885 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7886 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7887 }
7888 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7889 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7890
7891 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7892 X86IDTR16 IdtEntry;
7893 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7894 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7895 AssertRCReturn(rc, rc);
7896
7897 /* Construct the stack frame for the interrupt/exception handler. */
7898 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7899 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7900 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7901 AssertRCReturn(rc, rc);
7902
7903 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7904 if (rc == VINF_SUCCESS)
7905 {
7906 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7907 pMixedCtx->rip = IdtEntry.offSel;
7908 pMixedCtx->cs.Sel = IdtEntry.uSel;
7909 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7910 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7911 && uVector == X86_XCPT_PF)
7912 {
7913 pMixedCtx->cr2 = GCPtrFaultAddress;
7914 }
7915
7916 /* If any other guest-state bits are changed here, make sure to update
7917 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7918 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7919 | HM_CHANGED_GUEST_RIP
7920 | HM_CHANGED_GUEST_RFLAGS
7921 | HM_CHANGED_GUEST_RSP);
7922
7923 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7924 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7925 {
7926 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7927 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7928 Log4(("Clearing inhibition due to STI.\n"));
7929 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7930 }
7931 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7932
7933 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7934 it, if we are returning to ring-3 before executing guest code. */
7935 pVCpu->hm.s.Event.fPending = false;
7936 }
7937 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7938 return rc;
7939 }
7940 else
7941 {
7942 /*
7943 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7944 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7945 */
7946 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7947 }
7948 }
7949
7950 /* Validate. */
7951 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7952 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7953 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7954
7955 /* Inject. */
7956 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7957 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7958 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7959 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7960
7961 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7962 && uVector == X86_XCPT_PF)
7963 {
7964 pMixedCtx->cr2 = GCPtrFaultAddress;
7965 }
7966
7967 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7968 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7969
7970 AssertRCReturn(rc, rc);
7971 return rc;
7972}
7973
7974
7975/**
7976 * Clears the interrupt-window exiting control in the VMCS and if necessary
7977 * clears the current event in the VMCS as well.
7978 *
7979 * @returns VBox status code.
7980 * @param pVCpu Pointer to the VMCPU.
7981 *
7982 * @remarks Use this function only to clear events that have not yet been
7983 * delivered to the guest but are injected in the VMCS!
7984 * @remarks No-long-jump zone!!!
7985 */
7986static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7987{
7988 int rc;
7989 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7990
7991 /* Clear interrupt-window exiting control. */
7992 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7993 {
7994 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7995 Assert(!pVCpu->hm.s.Event.fPending);
7996 }
7997
7998 /* Clear NMI-window exiting control. */
7999 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8000 {
8001 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8002 Assert(!pVCpu->hm.s.Event.fPending);
8003 }
8004
8005 if (!pVCpu->hm.s.Event.fPending)
8006 return;
8007
8008#ifdef VBOX_STRICT
8009 uint32_t u32EntryInfo;
8010 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8011 AssertRC(rc);
8012 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8013#endif
8014
8015 /* Clear the entry-interruption field (including the valid bit). */
8016 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8017 AssertRC(rc);
8018
8019 /* Clear the pending debug exception field. */
8020 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8021 AssertRC(rc);
8022
8023 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8024 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8025}
8026
8027
8028/**
8029 * Enters the VT-x session.
8030 *
8031 * @returns VBox status code.
8032 * @param pVM Pointer to the VM.
8033 * @param pVCpu Pointer to the VMCPU.
8034 * @param pCpu Pointer to the CPU info struct.
8035 */
8036VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8037{
8038 AssertPtr(pVM);
8039 AssertPtr(pVCpu);
8040 Assert(pVM->hm.s.vmx.fSupported);
8041 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8042 NOREF(pCpu); NOREF(pVM);
8043
8044 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8045 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8046
8047#ifdef VBOX_STRICT
8048 /* Make sure we're in VMX root mode. */
8049 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8050 if (!(u32HostCR4 & X86_CR4_VMXE))
8051 {
8052 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8053 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8054 }
8055#endif
8056
8057 /*
8058 * Load the VCPU's VMCS as the current (and active) one.
8059 */
8060 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8061 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8062 if (RT_FAILURE(rc))
8063 return rc;
8064
8065 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8066 pVCpu->hm.s.fLeaveDone = false;
8067 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8068
8069 return VINF_SUCCESS;
8070}
8071
8072
8073/**
8074 * The thread-context callback (only on platforms which support it).
8075 *
8076 * @param enmEvent The thread-context event.
8077 * @param pVCpu Pointer to the VMCPU.
8078 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8079 * @thread EMT(pVCpu)
8080 */
8081VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8082{
8083 NOREF(fGlobalInit);
8084
8085 switch (enmEvent)
8086 {
8087 case RTTHREADCTXEVENT_PREEMPTING:
8088 {
8089 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8090 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8091 VMCPU_ASSERT_EMT(pVCpu);
8092
8093 PVM pVM = pVCpu->CTX_SUFF(pVM);
8094 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8095
8096 /* No longjmps (logger flushes, locks) in this fragile context. */
8097 VMMRZCallRing3Disable(pVCpu);
8098 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8099
8100 /*
8101 * Restore host-state (FPU, debug etc.)
8102 */
8103 if (!pVCpu->hm.s.fLeaveDone)
8104 {
8105 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8106 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8107 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8108 pVCpu->hm.s.fLeaveDone = true;
8109 }
8110
8111 /* Leave HM context, takes care of local init (term). */
8112 int rc = HMR0LeaveCpu(pVCpu);
8113 AssertRC(rc); NOREF(rc);
8114
8115 /* Restore longjmp state. */
8116 VMMRZCallRing3Enable(pVCpu);
8117 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8118 break;
8119 }
8120
8121 case RTTHREADCTXEVENT_RESUMED:
8122 {
8123 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8124 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8125 VMCPU_ASSERT_EMT(pVCpu);
8126
8127 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8128 VMMRZCallRing3Disable(pVCpu);
8129 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8130
8131 /* Initialize the bare minimum state required for HM. This takes care of
8132 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8133 int rc = HMR0EnterCpu(pVCpu);
8134 AssertRC(rc);
8135 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8136
8137 /* Load the active VMCS as the current one. */
8138 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8139 {
8140 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8141 AssertRC(rc); NOREF(rc);
8142 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8143 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8144 }
8145 pVCpu->hm.s.fLeaveDone = false;
8146
8147 /* Restore longjmp state. */
8148 VMMRZCallRing3Enable(pVCpu);
8149 break;
8150 }
8151
8152 default:
8153 break;
8154 }
8155}
8156
8157
8158/**
8159 * Saves the host state in the VMCS host-state.
8160 * Sets up the VM-exit MSR-load area.
8161 *
8162 * The CPU state will be loaded from these fields on every successful VM-exit.
8163 *
8164 * @returns VBox status code.
8165 * @param pVM Pointer to the VM.
8166 * @param pVCpu Pointer to the VMCPU.
8167 *
8168 * @remarks No-long-jump zone!!!
8169 */
8170static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8171{
8172 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8173
8174 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8175 return VINF_SUCCESS;
8176
8177 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8178 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8179
8180 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8181 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8182
8183 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8184 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8185
8186 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8187 return rc;
8188}
8189
8190
8191/**
8192 * Saves the host state in the VMCS host-state.
8193 *
8194 * @returns VBox status code.
8195 * @param pVM Pointer to the VM.
8196 * @param pVCpu Pointer to the VMCPU.
8197 *
8198 * @remarks No-long-jump zone!!!
8199 */
8200VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8201{
8202 AssertPtr(pVM);
8203 AssertPtr(pVCpu);
8204
8205 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8206
8207 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8208 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8209 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8210 return hmR0VmxSaveHostState(pVM, pVCpu);
8211}
8212
8213
8214/**
8215 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8216 * loaded from these fields on every successful VM-entry.
8217 *
8218 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8219 * Sets up the VM-entry controls.
8220 * Sets up the appropriate VMX non-root function to execute guest code based on
8221 * the guest CPU mode.
8222 *
8223 * @returns VBox status code.
8224 * @param pVM Pointer to the VM.
8225 * @param pVCpu Pointer to the VMCPU.
8226 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8227 * out-of-sync. Make sure to update the required fields
8228 * before using them.
8229 *
8230 * @remarks No-long-jump zone!!!
8231 */
8232static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8233{
8234 AssertPtr(pVM);
8235 AssertPtr(pVCpu);
8236 AssertPtr(pMixedCtx);
8237 HMVMX_ASSERT_PREEMPT_SAFE();
8238
8239#ifdef LOG_ENABLED
8240 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8241 * probably not initialized yet? Anyway this will do for now.
8242 *
8243 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8244 * interface and disable ring-3 calls when thread-context hooks are not
8245 * available. */
8246 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8247 VMMR0LogFlushDisable(pVCpu);
8248#endif
8249
8250 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8251
8252 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8253
8254 /* Determine real-on-v86 mode. */
8255 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8256 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8257 && CPUMIsGuestInRealModeEx(pMixedCtx))
8258 {
8259 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8260 }
8261
8262 /*
8263 * Load the guest-state into the VMCS.
8264 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8265 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8266 */
8267 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8268 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8269
8270 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8271 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8272 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8273
8274 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8275 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8276 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8277
8278 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8279 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8280
8281 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8282 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8283
8284 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8285 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8286 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8287
8288 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8289 determine we don't have to swap EFER after all. */
8290 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8291 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8292
8293 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8294 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8295
8296 /*
8297 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8298 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8299 */
8300 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8301 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8302
8303 /* Clear any unused and reserved bits. */
8304 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8305
8306#ifdef LOG_ENABLED
8307 /* Only reenable log-flushing if the caller has it enabled. */
8308 if (!fCallerDisabledLogFlush)
8309 VMMR0LogFlushEnable(pVCpu);
8310#endif
8311
8312 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8313 return rc;
8314}
8315
8316
8317/**
8318 * Loads the state shared between the host and guest into the VMCS.
8319 *
8320 * @param pVM Pointer to the VM.
8321 * @param pVCpu Pointer to the VMCPU.
8322 * @param pCtx Pointer to the guest-CPU context.
8323 *
8324 * @remarks No-long-jump zone!!!
8325 */
8326static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8327{
8328 NOREF(pVM);
8329
8330 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8331 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8332
8333 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8334 {
8335 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8336 AssertRC(rc);
8337 }
8338
8339 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8340 {
8341 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8342 AssertRC(rc);
8343
8344 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8345 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8346 {
8347 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8348 AssertRC(rc);
8349 }
8350 }
8351
8352 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8353 {
8354#if HC_ARCH_BITS == 64
8355 if (pVM->hm.s.fAllow64BitGuests)
8356 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8357#endif
8358 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8359 }
8360
8361 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8362 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8363}
8364
8365
8366/**
8367 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8368 *
8369 * @param pVM Pointer to the VM.
8370 * @param pVCpu Pointer to the VMCPU.
8371 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8372 * out-of-sync. Make sure to update the required fields
8373 * before using them.
8374 */
8375DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8376{
8377 HMVMX_ASSERT_PREEMPT_SAFE();
8378
8379 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8380#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8381 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8382#endif
8383
8384 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8385 {
8386 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8387 AssertRC(rc);
8388 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8389 }
8390 else if (HMCPU_CF_VALUE(pVCpu))
8391 {
8392 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8393 AssertRC(rc);
8394 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8395 }
8396
8397 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8398 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8399 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8400 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8401}
8402
8403
8404/**
8405 * Does the preparations before executing guest code in VT-x.
8406 *
8407 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8408 * recompiler. We must be cautious what we do here regarding committing
8409 * guest-state information into the VMCS assuming we assuredly execute the
8410 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8411 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8412 * so that the recompiler can (and should) use them when it resumes guest
8413 * execution. Otherwise such operations must be done when we can no longer
8414 * exit to ring-3.
8415 *
8416 * @returns Strict VBox status code.
8417 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8418 * have been disabled.
8419 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8420 * double-fault into the guest.
8421 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8422 *
8423 * @param pVM Pointer to the VM.
8424 * @param pVCpu Pointer to the VMCPU.
8425 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8426 * out-of-sync. Make sure to update the required fields
8427 * before using them.
8428 * @param pVmxTransient Pointer to the VMX transient structure.
8429 */
8430static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8431{
8432 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8433
8434#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8435 PGMRZDynMapFlushAutoSet(pVCpu);
8436#endif
8437
8438 /* Check force flag actions that might require us to go back to ring-3. */
8439 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8440 if (rc != VINF_SUCCESS)
8441 return rc;
8442
8443#ifndef IEM_VERIFICATION_MODE_FULL
8444 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8445 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8446 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8447 {
8448 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8449 RTGCPHYS GCPhysApicBase;
8450 GCPhysApicBase = pMixedCtx->msrApicBase;
8451 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8452
8453 /* Unalias any existing mapping. */
8454 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8455 AssertRCReturn(rc, rc);
8456
8457 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8458 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8459 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8460 AssertRCReturn(rc, rc);
8461
8462 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8463 }
8464#endif /* !IEM_VERIFICATION_MODE_FULL */
8465
8466 if (TRPMHasTrap(pVCpu))
8467 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8468 else if (!pVCpu->hm.s.Event.fPending)
8469 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8470
8471 /*
8472 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8473 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8474 */
8475 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8476 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8477 {
8478 Assert(rc == VINF_EM_RESET);
8479 return rc;
8480 }
8481
8482 /*
8483 * Load the guest state bits, we can handle longjmps/getting preempted here.
8484 *
8485 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8486 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8487 * Hence, this needs to be done -after- injection of events.
8488 */
8489 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8490
8491 /*
8492 * No longjmps to ring-3 from this point on!!!
8493 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8494 * This also disables flushing of the R0-logger instance (if any).
8495 */
8496 VMMRZCallRing3Disable(pVCpu);
8497
8498 /*
8499 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8500 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8501 *
8502 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8503 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8504 *
8505 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8506 * executing guest code.
8507 */
8508 pVmxTransient->uEflags = ASMIntDisableFlags();
8509 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8510 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8511 {
8512 hmR0VmxClearEventVmcs(pVCpu);
8513 ASMSetFlags(pVmxTransient->uEflags);
8514 VMMRZCallRing3Enable(pVCpu);
8515 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8516 return VINF_EM_RAW_TO_R3;
8517 }
8518
8519 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8520 {
8521 hmR0VmxClearEventVmcs(pVCpu);
8522 ASMSetFlags(pVmxTransient->uEflags);
8523 VMMRZCallRing3Enable(pVCpu);
8524 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8525 return VINF_EM_RAW_INTERRUPT;
8526 }
8527
8528 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8529 pVCpu->hm.s.Event.fPending = false;
8530
8531 return VINF_SUCCESS;
8532}
8533
8534
8535/**
8536 * Prepares to run guest code in VT-x and we've committed to doing so. This
8537 * means there is no backing out to ring-3 or anywhere else at this
8538 * point.
8539 *
8540 * @param pVM Pointer to the VM.
8541 * @param pVCpu Pointer to the VMCPU.
8542 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8543 * out-of-sync. Make sure to update the required fields
8544 * before using them.
8545 * @param pVmxTransient Pointer to the VMX transient structure.
8546 *
8547 * @remarks Called with preemption disabled.
8548 * @remarks No-long-jump zone!!!
8549 */
8550static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8551{
8552 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8553 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8554 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8555
8556 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8557 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8558
8559#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8560 if (!CPUMIsGuestFPUStateActive(pVCpu))
8561 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8562 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8563#endif
8564
8565 if ( pVCpu->hm.s.fUseGuestFpu
8566 && !CPUMIsGuestFPUStateActive(pVCpu))
8567 {
8568 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8569 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8570 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8571 }
8572
8573 /*
8574 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8575 */
8576 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8577 && pVCpu->hm.s.vmx.cMsrs > 0)
8578 {
8579 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8580 }
8581
8582 /*
8583 * Load the host state bits as we may've been preempted (only happens when
8584 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8585 */
8586 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8587 {
8588 /* This ASSUMES that pfnStartVM has been set up already. */
8589 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8590 AssertRC(rc);
8591 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8592 }
8593 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8594
8595 /*
8596 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8597 */
8598 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8599 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8600 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8601
8602 /* Store status of the shared guest-host state at the time of VM-entry. */
8603#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8604 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8605 {
8606 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8607 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8608 }
8609 else
8610#endif
8611 {
8612 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8613 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8614 }
8615 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8616
8617 /*
8618 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8619 */
8620 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8621 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8622
8623 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8624 RTCPUID idCurrentCpu = pCpu->idCpu;
8625 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8626 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8627 {
8628 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8629 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8630 }
8631
8632 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8633 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8634 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8635 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8636
8637 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8638
8639 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8640 to start executing. */
8641
8642 /*
8643 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8644 */
8645 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8646 {
8647 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8648 {
8649 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8650 AssertRC(rc2);
8651 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8652 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8653 true /* fUpdateHostMsr */);
8654 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8655 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8656 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8657 }
8658 else
8659 {
8660 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8661 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8662 }
8663 }
8664
8665#ifdef VBOX_STRICT
8666 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8667 hmR0VmxCheckHostEferMsr(pVCpu);
8668 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8669#endif
8670#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8671 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8672 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8673 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8674#endif
8675}
8676
8677
8678/**
8679 * Performs some essential restoration of state after running guest code in
8680 * VT-x.
8681 *
8682 * @param pVM Pointer to the VM.
8683 * @param pVCpu Pointer to the VMCPU.
8684 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8685 * out-of-sync. Make sure to update the required fields
8686 * before using them.
8687 * @param pVmxTransient Pointer to the VMX transient structure.
8688 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8689 *
8690 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8691 *
8692 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8693 * unconditionally when it is safe to do so.
8694 */
8695static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8696{
8697 NOREF(pVM);
8698
8699 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8700
8701 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8702 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8703 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8704 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8705 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8706
8707 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8708 {
8709 /** @todo Find a way to fix hardcoding a guestimate. */
8710 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8711 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8712 }
8713
8714 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8715 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8716 Assert(!(ASMGetFlags() & X86_EFL_IF));
8717 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8718
8719#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8720 if (CPUMIsGuestFPUStateActive(pVCpu))
8721 {
8722 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8723 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8724 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8725 }
8726#endif
8727
8728#if HC_ARCH_BITS == 64
8729 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8730#endif
8731 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8732#ifdef VBOX_STRICT
8733 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8734#endif
8735 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8736 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8737
8738 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8739 uint32_t uExitReason;
8740 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8741 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8742 AssertRC(rc);
8743 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8744 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8745
8746 /* Update the VM-exit history array. */
8747 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8748
8749 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8750 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8751 {
8752 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8753 pVmxTransient->fVMEntryFailed));
8754 return;
8755 }
8756
8757 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8758 {
8759 /** @todo We can optimize this by only syncing with our force-flags when
8760 * really needed and keeping the VMCS state as it is for most
8761 * VM-exits. */
8762 /* Update the guest interruptibility-state from the VMCS. */
8763 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8764
8765#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8766 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8767 AssertRC(rc);
8768#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8769 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8770 AssertRC(rc);
8771#endif
8772
8773 /*
8774 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8775 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8776 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8777 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8778 */
8779 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8780 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8781 {
8782 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8783 AssertRC(rc);
8784 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8785 }
8786 }
8787}
8788
8789
8790/**
8791 * Runs the guest code using VT-x the normal way.
8792 *
8793 * @returns VBox status code.
8794 * @param pVM Pointer to the VM.
8795 * @param pVCpu Pointer to the VMCPU.
8796 * @param pCtx Pointer to the guest-CPU context.
8797 *
8798 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8799 */
8800static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8801{
8802 VMXTRANSIENT VmxTransient;
8803 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8804 int rc = VERR_INTERNAL_ERROR_5;
8805 uint32_t cLoops = 0;
8806
8807 for (;; cLoops++)
8808 {
8809 Assert(!HMR0SuspendPending());
8810 HMVMX_ASSERT_CPU_SAFE();
8811
8812 /* Preparatory work for running guest code, this may force us to return
8813 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8814 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8815 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8816 if (rc != VINF_SUCCESS)
8817 break;
8818
8819 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8820 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8821 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8822
8823 /* Restore any residual host-state and save any bits shared between host
8824 and guest into the guest-CPU state. Re-enables interrupts! */
8825 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8826
8827 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8828 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8829 {
8830 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8831 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8832 return rc;
8833 }
8834
8835 /* Handle the VM-exit. */
8836 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8837 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8838 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8839 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8840 HMVMX_START_EXIT_DISPATCH_PROF();
8841#ifdef HMVMX_USE_FUNCTION_TABLE
8842 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8843#else
8844 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8845#endif
8846 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8847 if (rc != VINF_SUCCESS)
8848 break;
8849 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8850 {
8851 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8852 rc = VINF_EM_RAW_INTERRUPT;
8853 break;
8854 }
8855 }
8856
8857 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8858 return rc;
8859}
8860
8861
8862/**
8863 * Single steps guest code using VT-x.
8864 *
8865 * @returns VBox status code.
8866 * @param pVM Pointer to the VM.
8867 * @param pVCpu Pointer to the VMCPU.
8868 * @param pCtx Pointer to the guest-CPU context.
8869 *
8870 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8871 */
8872static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8873{
8874 VMXTRANSIENT VmxTransient;
8875 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8876 int rc = VERR_INTERNAL_ERROR_5;
8877 uint32_t cLoops = 0;
8878 uint16_t uCsStart = pCtx->cs.Sel;
8879 uint64_t uRipStart = pCtx->rip;
8880
8881 for (;; cLoops++)
8882 {
8883 Assert(!HMR0SuspendPending());
8884 HMVMX_ASSERT_CPU_SAFE();
8885
8886 /* Preparatory work for running guest code, this may force us to return
8887 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8888 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8889 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8890 if (rc != VINF_SUCCESS)
8891 break;
8892
8893 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8894 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8895 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8896
8897 /* Restore any residual host-state and save any bits shared between host
8898 and guest into the guest-CPU state. Re-enables interrupts! */
8899 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8900
8901 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8902 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8903 {
8904 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8905 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8906 return rc;
8907 }
8908
8909 /* Handle the VM-exit. */
8910 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8911 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8912 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8913 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8914 HMVMX_START_EXIT_DISPATCH_PROF();
8915#ifdef HMVMX_USE_FUNCTION_TABLE
8916 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8917#else
8918 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8919#endif
8920 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8921 if (rc != VINF_SUCCESS)
8922 break;
8923 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8924 {
8925 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8926 rc = VINF_EM_RAW_INTERRUPT;
8927 break;
8928 }
8929
8930 /*
8931 * Did the RIP change, if so, consider it a single step.
8932 * Otherwise, make sure one of the TFs gets set.
8933 */
8934 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8935 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8936 AssertRCReturn(rc2, rc2);
8937 if ( pCtx->rip != uRipStart
8938 || pCtx->cs.Sel != uCsStart)
8939 {
8940 rc = VINF_EM_DBG_STEPPED;
8941 break;
8942 }
8943 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8944 }
8945
8946 /*
8947 * Clear the X86_EFL_TF if necessary.
8948 */
8949 if (pVCpu->hm.s.fClearTrapFlag)
8950 {
8951 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8952 AssertRCReturn(rc2, rc2);
8953 pVCpu->hm.s.fClearTrapFlag = false;
8954 pCtx->eflags.Bits.u1TF = 0;
8955 }
8956 /** @todo there seems to be issues with the resume flag when the monitor trap
8957 * flag is pending without being used. Seen early in bios init when
8958 * accessing APIC page in protected mode. */
8959
8960 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8961 return rc;
8962}
8963
8964
8965/**
8966 * Runs the guest code using VT-x.
8967 *
8968 * @returns VBox status code.
8969 * @param pVM Pointer to the VM.
8970 * @param pVCpu Pointer to the VMCPU.
8971 * @param pCtx Pointer to the guest-CPU context.
8972 */
8973VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8974{
8975 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8976 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8977 HMVMX_ASSERT_PREEMPT_SAFE();
8978
8979 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8980
8981 int rc;
8982 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8983 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8984 else
8985 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8986
8987 if (rc == VERR_EM_INTERPRETER)
8988 rc = VINF_EM_RAW_EMULATE_INSTR;
8989 else if (rc == VINF_EM_RESET)
8990 rc = VINF_EM_TRIPLE_FAULT;
8991
8992 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8993 if (RT_FAILURE(rc2))
8994 {
8995 pVCpu->hm.s.u32HMError = rc;
8996 rc = rc2;
8997 }
8998 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8999 return rc;
9000}
9001
9002
9003#ifndef HMVMX_USE_FUNCTION_TABLE
9004DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9005{
9006#ifdef DEBUG_ramshankar
9007# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9008# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9009#endif
9010 int rc;
9011 switch (rcReason)
9012 {
9013 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9014 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9015 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9016 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9017 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9018 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9019 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9020 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9021 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9022 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9023 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9024 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9025 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9026 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9027 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9028 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9029 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9042 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9043 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9044 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9045 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9046 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9047
9048 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9049 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9050 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9051 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9052 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9053 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9054 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9055 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9056 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9057
9058 case VMX_EXIT_VMCLEAR:
9059 case VMX_EXIT_VMLAUNCH:
9060 case VMX_EXIT_VMPTRLD:
9061 case VMX_EXIT_VMPTRST:
9062 case VMX_EXIT_VMREAD:
9063 case VMX_EXIT_VMRESUME:
9064 case VMX_EXIT_VMWRITE:
9065 case VMX_EXIT_VMXOFF:
9066 case VMX_EXIT_VMXON:
9067 case VMX_EXIT_INVEPT:
9068 case VMX_EXIT_INVVPID:
9069 case VMX_EXIT_VMFUNC:
9070 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9071 break;
9072 default:
9073 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9074 break;
9075 }
9076 return rc;
9077}
9078#endif
9079
9080#ifdef DEBUG
9081/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9082# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9083 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9084
9085# define HMVMX_ASSERT_PREEMPT_CPUID() \
9086 do \
9087 { \
9088 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9089 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9090 } while (0)
9091
9092# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9093 do { \
9094 AssertPtr(pVCpu); \
9095 AssertPtr(pMixedCtx); \
9096 AssertPtr(pVmxTransient); \
9097 Assert(pVmxTransient->fVMEntryFailed == false); \
9098 Assert(ASMIntAreEnabled()); \
9099 HMVMX_ASSERT_PREEMPT_SAFE(); \
9100 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9101 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9102 HMVMX_ASSERT_PREEMPT_SAFE(); \
9103 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9104 HMVMX_ASSERT_PREEMPT_CPUID(); \
9105 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9106 } while (0)
9107
9108# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9109 do { \
9110 Log4Func(("\n")); \
9111 } while (0)
9112#else /* Release builds */
9113# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9114 do { \
9115 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9116 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9117 } while (0)
9118# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9119#endif
9120
9121
9122/**
9123 * Advances the guest RIP after reading it from the VMCS.
9124 *
9125 * @returns VBox status code.
9126 * @param pVCpu Pointer to the VMCPU.
9127 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9128 * out-of-sync. Make sure to update the required fields
9129 * before using them.
9130 * @param pVmxTransient Pointer to the VMX transient structure.
9131 *
9132 * @remarks No-long-jump zone!!!
9133 */
9134DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9135{
9136 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9137 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9138 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9139 AssertRCReturn(rc, rc);
9140
9141 pMixedCtx->rip += pVmxTransient->cbInstr;
9142 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9143
9144 /*
9145 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9146 * pending debug exception field as it takes care of priority of events.
9147 *
9148 * See Intel spec. 32.2.1 "Debug Exceptions".
9149 */
9150 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9151
9152 return rc;
9153}
9154
9155
9156/**
9157 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9158 * and update error record fields accordingly.
9159 *
9160 * @return VMX_IGS_* return codes.
9161 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9162 * wrong with the guest state.
9163 *
9164 * @param pVM Pointer to the VM.
9165 * @param pVCpu Pointer to the VMCPU.
9166 * @param pCtx Pointer to the guest-CPU state.
9167 *
9168 * @remarks This function assumes our cache of the VMCS controls
9169 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9170 */
9171static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9172{
9173#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9174#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9175 uError = (err); \
9176 break; \
9177 } else do { } while (0)
9178
9179 int rc;
9180 uint32_t uError = VMX_IGS_ERROR;
9181 uint32_t u32Val;
9182 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9183
9184 do
9185 {
9186 /*
9187 * CR0.
9188 */
9189 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9190 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9191 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9192 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9193 if (fUnrestrictedGuest)
9194 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9195
9196 uint32_t u32GuestCR0;
9197 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9198 AssertRCBreak(rc);
9199 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9200 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9201 if ( !fUnrestrictedGuest
9202 && (u32GuestCR0 & X86_CR0_PG)
9203 && !(u32GuestCR0 & X86_CR0_PE))
9204 {
9205 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9206 }
9207
9208 /*
9209 * CR4.
9210 */
9211 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9212 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9213
9214 uint32_t u32GuestCR4;
9215 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9216 AssertRCBreak(rc);
9217 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9218 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9219
9220 /*
9221 * IA32_DEBUGCTL MSR.
9222 */
9223 uint64_t u64Val;
9224 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9225 AssertRCBreak(rc);
9226 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9227 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9228 {
9229 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9230 }
9231 uint64_t u64DebugCtlMsr = u64Val;
9232
9233#ifdef VBOX_STRICT
9234 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9235 AssertRCBreak(rc);
9236 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9237#endif
9238 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9239
9240 /*
9241 * RIP and RFLAGS.
9242 */
9243 uint32_t u32Eflags;
9244#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9245 if (HMVMX_IS_64BIT_HOST_MODE())
9246 {
9247 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9248 AssertRCBreak(rc);
9249 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9250 if ( !fLongModeGuest
9251 || !pCtx->cs.Attr.n.u1Long)
9252 {
9253 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9254 }
9255 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9256 * must be identical if the "IA-32e mode guest" VM-entry
9257 * control is 1 and CS.L is 1. No check applies if the
9258 * CPU supports 64 linear-address bits. */
9259
9260 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9261 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9262 AssertRCBreak(rc);
9263 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9264 VMX_IGS_RFLAGS_RESERVED);
9265 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9266 u32Eflags = u64Val;
9267 }
9268 else
9269#endif
9270 {
9271 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9272 AssertRCBreak(rc);
9273 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9274 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9275 }
9276
9277 if ( fLongModeGuest
9278 || ( fUnrestrictedGuest
9279 && !(u32GuestCR0 & X86_CR0_PE)))
9280 {
9281 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9282 }
9283
9284 uint32_t u32EntryInfo;
9285 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9286 AssertRCBreak(rc);
9287 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9288 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9289 {
9290 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9291 }
9292
9293 /*
9294 * 64-bit checks.
9295 */
9296#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9297 if (HMVMX_IS_64BIT_HOST_MODE())
9298 {
9299 if ( fLongModeGuest
9300 && !fUnrestrictedGuest)
9301 {
9302 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9303 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9304 }
9305
9306 if ( !fLongModeGuest
9307 && (u32GuestCR4 & X86_CR4_PCIDE))
9308 {
9309 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9310 }
9311
9312 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9313 * 51:32 beyond the processor's physical-address width are 0. */
9314
9315 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9316 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9317 {
9318 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9319 }
9320
9321 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9322 AssertRCBreak(rc);
9323 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9324
9325 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9326 AssertRCBreak(rc);
9327 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9328 }
9329#endif
9330
9331 /*
9332 * PERF_GLOBAL MSR.
9333 */
9334 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9335 {
9336 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9337 AssertRCBreak(rc);
9338 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9339 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9340 }
9341
9342 /*
9343 * PAT MSR.
9344 */
9345 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9346 {
9347 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9348 AssertRCBreak(rc);
9349 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9350 for (unsigned i = 0; i < 8; i++)
9351 {
9352 uint8_t u8Val = (u64Val & 0x7);
9353 if ( u8Val != 0 /* UC */
9354 || u8Val != 1 /* WC */
9355 || u8Val != 4 /* WT */
9356 || u8Val != 5 /* WP */
9357 || u8Val != 6 /* WB */
9358 || u8Val != 7 /* UC- */)
9359 {
9360 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9361 }
9362 u64Val >>= 3;
9363 }
9364 }
9365
9366 /*
9367 * EFER MSR.
9368 */
9369 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9370 {
9371 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9372 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9373 AssertRCBreak(rc);
9374 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9375 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9376 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9377 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9378 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9379 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9380 VMX_IGS_EFER_LMA_PG_MISMATCH);
9381 }
9382
9383 /*
9384 * Segment registers.
9385 */
9386 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9387 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9388 if (!(u32Eflags & X86_EFL_VM))
9389 {
9390 /* CS */
9391 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9392 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9393 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9394 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9395 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9396 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9397 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9398 /* CS cannot be loaded with NULL in protected mode. */
9399 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9400 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9401 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9402 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9403 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9404 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9405 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9406 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9407 else
9408 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9409
9410 /* SS */
9411 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9412 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9413 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9414 if ( !(pCtx->cr0 & X86_CR0_PE)
9415 || pCtx->cs.Attr.n.u4Type == 3)
9416 {
9417 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9418 }
9419 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9420 {
9421 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9422 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9423 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9424 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9425 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9426 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9427 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9428 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9429 }
9430
9431 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9432 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9433 {
9434 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9435 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9436 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9437 || pCtx->ds.Attr.n.u4Type > 11
9438 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9439 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9440 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9441 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9442 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9443 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9444 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9445 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9446 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9447 }
9448 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9449 {
9450 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9451 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9452 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9453 || pCtx->es.Attr.n.u4Type > 11
9454 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9455 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9456 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9457 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9458 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9459 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9460 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9461 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9462 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9463 }
9464 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9465 {
9466 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9467 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9468 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9469 || pCtx->fs.Attr.n.u4Type > 11
9470 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9471 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9472 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9473 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9474 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9475 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9476 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9477 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9478 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9479 }
9480 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9481 {
9482 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9483 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9484 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9485 || pCtx->gs.Attr.n.u4Type > 11
9486 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9487 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9488 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9489 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9490 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9491 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9492 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9493 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9494 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9495 }
9496 /* 64-bit capable CPUs. */
9497#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9498 if (HMVMX_IS_64BIT_HOST_MODE())
9499 {
9500 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9501 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9502 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9503 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9504 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9505 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9506 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9507 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9508 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9509 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9510 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9511 }
9512#endif
9513 }
9514 else
9515 {
9516 /* V86 mode checks. */
9517 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9518 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9519 {
9520 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9521 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9522 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9523 }
9524 else
9525 {
9526 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9527 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9528 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9529 }
9530
9531 /* CS */
9532 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9533 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9534 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9535 /* SS */
9536 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9537 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9538 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9539 /* DS */
9540 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9541 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9542 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9543 /* ES */
9544 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9545 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9546 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9547 /* FS */
9548 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9549 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9550 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9551 /* GS */
9552 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9553 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9554 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9555 /* 64-bit capable CPUs. */
9556#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9557 if (HMVMX_IS_64BIT_HOST_MODE())
9558 {
9559 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9560 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9561 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9562 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9563 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9564 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9565 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9566 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9567 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9568 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9569 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9570 }
9571#endif
9572 }
9573
9574 /*
9575 * TR.
9576 */
9577 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9578 /* 64-bit capable CPUs. */
9579#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9580 if (HMVMX_IS_64BIT_HOST_MODE())
9581 {
9582 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9583 }
9584#endif
9585 if (fLongModeGuest)
9586 {
9587 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9588 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9589 }
9590 else
9591 {
9592 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9593 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9594 VMX_IGS_TR_ATTR_TYPE_INVALID);
9595 }
9596 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9597 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9598 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9599 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9600 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9601 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9602 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9603 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9604
9605 /*
9606 * GDTR and IDTR.
9607 */
9608#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9609 if (HMVMX_IS_64BIT_HOST_MODE())
9610 {
9611 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9612 AssertRCBreak(rc);
9613 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9614
9615 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9616 AssertRCBreak(rc);
9617 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9618 }
9619#endif
9620
9621 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9622 AssertRCBreak(rc);
9623 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9624
9625 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9626 AssertRCBreak(rc);
9627 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9628
9629 /*
9630 * Guest Non-Register State.
9631 */
9632 /* Activity State. */
9633 uint32_t u32ActivityState;
9634 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9635 AssertRCBreak(rc);
9636 HMVMX_CHECK_BREAK( !u32ActivityState
9637 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9638 VMX_IGS_ACTIVITY_STATE_INVALID);
9639 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9640 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9641 uint32_t u32IntrState;
9642 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9643 AssertRCBreak(rc);
9644 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9645 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9646 {
9647 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9648 }
9649
9650 /** @todo Activity state and injecting interrupts. Left as a todo since we
9651 * currently don't use activity states but ACTIVE. */
9652
9653 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9654 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9655
9656 /* Guest interruptibility-state. */
9657 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9658 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9659 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9660 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9661 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9662 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9663 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9664 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9665 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9666 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9667 {
9668 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9669 {
9670 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9671 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9672 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9673 }
9674 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9675 {
9676 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9677 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9678 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9679 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9680 }
9681 }
9682 /** @todo Assumes the processor is not in SMM. */
9683 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9684 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9685 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9686 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9687 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9688 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9689 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9690 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9691 {
9692 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9693 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9694 }
9695
9696 /* Pending debug exceptions. */
9697 if (HMVMX_IS_64BIT_HOST_MODE())
9698 {
9699 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9700 AssertRCBreak(rc);
9701 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9702 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9703 u32Val = u64Val; /* For pending debug exceptions checks below. */
9704 }
9705 else
9706 {
9707 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9708 AssertRCBreak(rc);
9709 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9710 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9711 }
9712
9713 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9714 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9715 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9716 {
9717 if ( (u32Eflags & X86_EFL_TF)
9718 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9719 {
9720 /* Bit 14 is PendingDebug.BS. */
9721 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9722 }
9723 if ( !(u32Eflags & X86_EFL_TF)
9724 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9725 {
9726 /* Bit 14 is PendingDebug.BS. */
9727 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9728 }
9729 }
9730
9731 /* VMCS link pointer. */
9732 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9733 AssertRCBreak(rc);
9734 if (u64Val != UINT64_C(0xffffffffffffffff))
9735 {
9736 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9737 /** @todo Bits beyond the processor's physical-address width MBZ. */
9738 /** @todo 32-bit located in memory referenced by value of this field (as a
9739 * physical address) must contain the processor's VMCS revision ID. */
9740 /** @todo SMM checks. */
9741 }
9742
9743 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9744 * not using Nested Paging? */
9745 if ( pVM->hm.s.fNestedPaging
9746 && !fLongModeGuest
9747 && CPUMIsGuestInPAEModeEx(pCtx))
9748 {
9749 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9750 AssertRCBreak(rc);
9751 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9752
9753 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9754 AssertRCBreak(rc);
9755 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9756
9757 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9758 AssertRCBreak(rc);
9759 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9760
9761 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9762 AssertRCBreak(rc);
9763 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9764 }
9765
9766 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9767 if (uError == VMX_IGS_ERROR)
9768 uError = VMX_IGS_REASON_NOT_FOUND;
9769 } while (0);
9770
9771 pVCpu->hm.s.u32HMError = uError;
9772 return uError;
9773
9774#undef HMVMX_ERROR_BREAK
9775#undef HMVMX_CHECK_BREAK
9776}
9777
9778/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9779/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9780/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9781
9782/** @name VM-exit handlers.
9783 * @{
9784 */
9785
9786/**
9787 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9788 */
9789HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9790{
9791 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9792 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9793 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9794 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9795 return VINF_SUCCESS;
9796 return VINF_EM_RAW_INTERRUPT;
9797}
9798
9799
9800/**
9801 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9802 */
9803HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9804{
9805 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9806 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9807
9808 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9809 AssertRCReturn(rc, rc);
9810
9811 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9812 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9813 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9814 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9815
9816 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9817 {
9818 /*
9819 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9820 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9821 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9822 *
9823 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9824 */
9825 VMXDispatchHostNmi();
9826 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9827 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9828 return VINF_SUCCESS;
9829 }
9830
9831 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9832 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9833 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9834 {
9835 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9836 return VINF_SUCCESS;
9837 }
9838 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9839 {
9840 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9841 return rc;
9842 }
9843
9844 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9845 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9846 switch (uIntType)
9847 {
9848 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9849 Assert(uVector == X86_XCPT_DB);
9850 /* no break */
9851 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9852 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9853 /* no break */
9854 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9855 {
9856 switch (uVector)
9857 {
9858 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9859 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9860 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9861 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9862 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9863 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9864#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9865 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9866 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9867 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9868 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9869 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9870 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9871 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9872 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9873 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9874 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9875 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9876 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9877#endif
9878 default:
9879 {
9880 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9881 AssertRCReturn(rc, rc);
9882
9883 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9884 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9885 {
9886 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9887 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9888 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9889
9890 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9891 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9892 AssertRCReturn(rc, rc);
9893 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9894 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9895 0 /* GCPtrFaultAddress */);
9896 AssertRCReturn(rc, rc);
9897 }
9898 else
9899 {
9900 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9901 pVCpu->hm.s.u32HMError = uVector;
9902 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9903 }
9904 break;
9905 }
9906 }
9907 break;
9908 }
9909
9910 default:
9911 {
9912 pVCpu->hm.s.u32HMError = uExitIntInfo;
9913 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9914 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9915 break;
9916 }
9917 }
9918 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9919 return rc;
9920}
9921
9922
9923/**
9924 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9925 */
9926HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9927{
9928 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9929
9930 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9931 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9932
9933 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9934 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9935 return VINF_SUCCESS;
9936}
9937
9938
9939/**
9940 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9941 */
9942HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9943{
9944 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9945 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
9946 {
9947 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9948 HMVMX_RETURN_UNEXPECTED_EXIT();
9949 }
9950
9951 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
9952
9953 /*
9954 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
9955 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
9956 */
9957 uint32_t uIntrState = 0;
9958 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9959 AssertRCReturn(rc, rc);
9960
9961 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
9962 if ( fBlockSti
9963 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
9964 {
9965 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9966 }
9967
9968 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
9969 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
9970
9971 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9972 return VINF_SUCCESS;
9973}
9974
9975
9976/**
9977 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9978 */
9979HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9980{
9981 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9983 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9984}
9985
9986
9987/**
9988 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9989 */
9990HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9991{
9992 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9993 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9994 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9995}
9996
9997
9998/**
9999 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10000 */
10001HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10002{
10003 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10004 PVM pVM = pVCpu->CTX_SUFF(pVM);
10005 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10006 if (RT_LIKELY(rc == VINF_SUCCESS))
10007 {
10008 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10009 Assert(pVmxTransient->cbInstr == 2);
10010 }
10011 else
10012 {
10013 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10014 rc = VERR_EM_INTERPRETER;
10015 }
10016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10017 return rc;
10018}
10019
10020
10021/**
10022 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10023 */
10024HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10025{
10026 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10027 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10028 AssertRCReturn(rc, rc);
10029
10030 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10031 return VINF_EM_RAW_EMULATE_INSTR;
10032
10033 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10034 HMVMX_RETURN_UNEXPECTED_EXIT();
10035}
10036
10037
10038/**
10039 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10040 */
10041HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10042{
10043 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10044 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10045 AssertRCReturn(rc, rc);
10046
10047 PVM pVM = pVCpu->CTX_SUFF(pVM);
10048 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10049 if (RT_LIKELY(rc == VINF_SUCCESS))
10050 {
10051 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10052 Assert(pVmxTransient->cbInstr == 2);
10053 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10054 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10055 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10056 }
10057 else
10058 {
10059 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
10060 rc = VERR_EM_INTERPRETER;
10061 }
10062 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10063 return rc;
10064}
10065
10066
10067/**
10068 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10069 */
10070HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10071{
10072 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10073 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10074 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10075 AssertRCReturn(rc, rc);
10076
10077 PVM pVM = pVCpu->CTX_SUFF(pVM);
10078 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10079 if (RT_LIKELY(rc == VINF_SUCCESS))
10080 {
10081 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10082 Assert(pVmxTransient->cbInstr == 3);
10083 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10084 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10085 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10086 }
10087 else
10088 {
10089 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10090 rc = VERR_EM_INTERPRETER;
10091 }
10092 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10093 return rc;
10094}
10095
10096
10097/**
10098 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10099 */
10100HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10101{
10102 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10103 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10104 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10105 AssertRCReturn(rc, rc);
10106
10107 PVM pVM = pVCpu->CTX_SUFF(pVM);
10108 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10109 if (RT_LIKELY(rc == VINF_SUCCESS))
10110 {
10111 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10112 Assert(pVmxTransient->cbInstr == 2);
10113 }
10114 else
10115 {
10116 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10117 rc = VERR_EM_INTERPRETER;
10118 }
10119 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10120 return rc;
10121}
10122
10123
10124/**
10125 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10126 */
10127HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10128{
10129 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10130
10131 int rc = VERR_NOT_SUPPORTED;
10132 if (GIMAreHypercallsEnabled(pVCpu))
10133 {
10134 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10135 AssertRCReturn(rc, rc);
10136
10137 rc = GIMHypercall(pVCpu, pMixedCtx);
10138 }
10139 if (rc != VINF_SUCCESS)
10140 {
10141 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10142 rc = VINF_SUCCESS;
10143 }
10144
10145 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10146 return rc;
10147}
10148
10149
10150/**
10151 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10152 */
10153HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10154{
10155 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10156 PVM pVM = pVCpu->CTX_SUFF(pVM);
10157 Assert(!pVM->hm.s.fNestedPaging);
10158
10159 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10160 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10161 AssertRCReturn(rc, rc);
10162
10163 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10164 rc = VBOXSTRICTRC_VAL(rc2);
10165 if (RT_LIKELY(rc == VINF_SUCCESS))
10166 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10167 else
10168 {
10169 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10170 pVmxTransient->uExitQualification, rc));
10171 }
10172 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10173 return rc;
10174}
10175
10176
10177/**
10178 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10179 */
10180HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10181{
10182 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10183 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10184 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10185 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10186 AssertRCReturn(rc, rc);
10187
10188 PVM pVM = pVCpu->CTX_SUFF(pVM);
10189 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10190 if (RT_LIKELY(rc == VINF_SUCCESS))
10191 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10192 else
10193 {
10194 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10195 rc = VERR_EM_INTERPRETER;
10196 }
10197 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10198 return rc;
10199}
10200
10201
10202/**
10203 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10204 */
10205HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10206{
10207 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10208 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10209 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10210 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10211 AssertRCReturn(rc, rc);
10212
10213 PVM pVM = pVCpu->CTX_SUFF(pVM);
10214 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10215 rc = VBOXSTRICTRC_VAL(rc2);
10216 if (RT_LIKELY( rc == VINF_SUCCESS
10217 || rc == VINF_EM_HALT))
10218 {
10219 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10220 AssertRCReturn(rc3, rc3);
10221
10222 if ( rc == VINF_EM_HALT
10223 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10224 {
10225 rc = VINF_SUCCESS;
10226 }
10227 }
10228 else
10229 {
10230 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10231 rc = VERR_EM_INTERPRETER;
10232 }
10233 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10234 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10235 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10236 return rc;
10237}
10238
10239
10240/**
10241 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10242 */
10243HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10244{
10245 /*
10246 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10247 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10248 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10249 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10250 */
10251 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10252 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10253 HMVMX_RETURN_UNEXPECTED_EXIT();
10254}
10255
10256
10257/**
10258 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10259 */
10260HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10261{
10262 /*
10263 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10264 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10265 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10266 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10267 */
10268 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10269 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10270 HMVMX_RETURN_UNEXPECTED_EXIT();
10271}
10272
10273
10274/**
10275 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10276 */
10277HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10278{
10279 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10280 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10281 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10282 HMVMX_RETURN_UNEXPECTED_EXIT();
10283}
10284
10285
10286/**
10287 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10288 */
10289HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10290{
10291 /*
10292 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10293 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10294 * See Intel spec. 25.3 "Other Causes of VM-exits".
10295 */
10296 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10297 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10298 HMVMX_RETURN_UNEXPECTED_EXIT();
10299}
10300
10301
10302/**
10303 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10304 * VM-exit.
10305 */
10306HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10307{
10308 /*
10309 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10310 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10311 *
10312 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10313 * See Intel spec. "23.8 Restrictions on VMX operation".
10314 */
10315 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10316 return VINF_SUCCESS;
10317}
10318
10319
10320/**
10321 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10322 * VM-exit.
10323 */
10324HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10325{
10326 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10327 return VINF_EM_RESET;
10328}
10329
10330
10331/**
10332 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10333 */
10334HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10335{
10336 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10337 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10338 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10339 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10340 AssertRCReturn(rc, rc);
10341
10342 pMixedCtx->rip++;
10343 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10344 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10345 rc = VINF_SUCCESS;
10346 else
10347 rc = VINF_EM_HALT;
10348
10349 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10350 return rc;
10351}
10352
10353
10354/**
10355 * VM-exit handler for instructions that result in a #UD exception delivered to
10356 * the guest.
10357 */
10358HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10359{
10360 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10361 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10362 return VINF_SUCCESS;
10363}
10364
10365
10366/**
10367 * VM-exit handler for expiry of the VMX preemption timer.
10368 */
10369HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10370{
10371 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10372
10373 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10374 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10375
10376 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10377 PVM pVM = pVCpu->CTX_SUFF(pVM);
10378 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10379 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10380 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10381}
10382
10383
10384/**
10385 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10386 */
10387HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10388{
10389 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10390
10391 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10392 /** @todo check if XSETBV is supported by the recompiler. */
10393 return VERR_EM_INTERPRETER;
10394}
10395
10396
10397/**
10398 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10399 */
10400HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10401{
10402 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10403
10404 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10405 /** @todo implement EMInterpretInvpcid() */
10406 return VERR_EM_INTERPRETER;
10407}
10408
10409
10410/**
10411 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10412 * Error VM-exit.
10413 */
10414HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10415{
10416 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10417 AssertRCReturn(rc, rc);
10418
10419 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10420 AssertRCReturn(rc, rc);
10421
10422 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10423 NOREF(uInvalidReason);
10424
10425#ifdef VBOX_STRICT
10426 uint32_t uIntrState;
10427 HMVMXHCUINTREG uHCReg;
10428 uint64_t u64Val;
10429 uint32_t u32Val;
10430
10431 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10432 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10433 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10434 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10435 AssertRCReturn(rc, rc);
10436
10437 Log4(("uInvalidReason %u\n", uInvalidReason));
10438 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10439 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10440 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10441 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10442
10443 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10444 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10445 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10446 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10447 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10448 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10449 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10450 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10451 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10452 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10453 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10454 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10455#else
10456 NOREF(pVmxTransient);
10457#endif
10458
10459 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10460 return VERR_VMX_INVALID_GUEST_STATE;
10461}
10462
10463
10464/**
10465 * VM-exit handler for VM-entry failure due to an MSR-load
10466 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10467 */
10468HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10469{
10470 NOREF(pVmxTransient);
10471 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10472 HMVMX_RETURN_UNEXPECTED_EXIT();
10473}
10474
10475
10476/**
10477 * VM-exit handler for VM-entry failure due to a machine-check event
10478 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10479 */
10480HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10481{
10482 NOREF(pVmxTransient);
10483 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10484 HMVMX_RETURN_UNEXPECTED_EXIT();
10485}
10486
10487
10488/**
10489 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10490 * theory.
10491 */
10492HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10493{
10494 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10495 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10496 return VERR_VMX_UNDEFINED_EXIT_CODE;
10497}
10498
10499
10500/**
10501 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10502 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10503 * Conditional VM-exit.
10504 */
10505HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10506{
10507 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10508
10509 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10510 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10511 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10512 return VERR_EM_INTERPRETER;
10513 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10514 HMVMX_RETURN_UNEXPECTED_EXIT();
10515}
10516
10517
10518/**
10519 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10520 */
10521HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10522{
10523 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10524
10525 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10526 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10527 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10528 return VERR_EM_INTERPRETER;
10529 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10530 HMVMX_RETURN_UNEXPECTED_EXIT();
10531}
10532
10533
10534/**
10535 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10536 */
10537HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10538{
10539 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10540
10541 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10542 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10543 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10544 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10545 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10546 {
10547 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10548 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10549 }
10550 AssertRCReturn(rc, rc);
10551 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10552
10553#ifdef VBOX_STRICT
10554 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10555 {
10556 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10557 && pMixedCtx->ecx != MSR_K6_EFER)
10558 {
10559 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10560 HMVMX_RETURN_UNEXPECTED_EXIT();
10561 }
10562# if HC_ARCH_BITS == 64
10563 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10564 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10565 {
10566 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10567 HMVMX_RETURN_UNEXPECTED_EXIT();
10568 }
10569# endif
10570 }
10571#endif
10572
10573 PVM pVM = pVCpu->CTX_SUFF(pVM);
10574 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10575 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10576 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10577 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10578 if (RT_LIKELY(rc == VINF_SUCCESS))
10579 {
10580 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10581 Assert(pVmxTransient->cbInstr == 2);
10582 }
10583 return rc;
10584}
10585
10586
10587/**
10588 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10589 */
10590HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10591{
10592 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10593 PVM pVM = pVCpu->CTX_SUFF(pVM);
10594 int rc = VINF_SUCCESS;
10595
10596 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10597 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10598 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10599 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10600 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10601 {
10602 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10603 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10604 }
10605 AssertRCReturn(rc, rc);
10606 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10607
10608 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10609 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10610 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10611
10612 if (RT_LIKELY(rc == VINF_SUCCESS))
10613 {
10614 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10615
10616 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10617 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10618 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10619 {
10620 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10621 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10622 EMInterpretWrmsr() changes it. */
10623 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10624 }
10625 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10626 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10627 else if (pMixedCtx->ecx == MSR_K6_EFER)
10628 {
10629 /*
10630 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10631 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10632 * the other bits as well, SCE and NXE. See @bugref{7368}.
10633 */
10634 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10635 }
10636
10637 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10638 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10639 {
10640 switch (pMixedCtx->ecx)
10641 {
10642 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10643 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10644 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10645 case MSR_K8_FS_BASE: /* no break */
10646 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10647 case MSR_K6_EFER: /* already handled above */ break;
10648 default:
10649 {
10650 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10651 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10652#if HC_ARCH_BITS == 64
10653 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10654 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10655#endif
10656 break;
10657 }
10658 }
10659 }
10660#ifdef VBOX_STRICT
10661 else
10662 {
10663 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10664 switch (pMixedCtx->ecx)
10665 {
10666 case MSR_IA32_SYSENTER_CS:
10667 case MSR_IA32_SYSENTER_EIP:
10668 case MSR_IA32_SYSENTER_ESP:
10669 case MSR_K8_FS_BASE:
10670 case MSR_K8_GS_BASE:
10671 {
10672 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10673 HMVMX_RETURN_UNEXPECTED_EXIT();
10674 }
10675
10676 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10677 default:
10678 {
10679 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10680 {
10681 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10682 if (pMixedCtx->ecx != MSR_K6_EFER)
10683 {
10684 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10685 pMixedCtx->ecx));
10686 HMVMX_RETURN_UNEXPECTED_EXIT();
10687 }
10688 }
10689
10690#if HC_ARCH_BITS == 64
10691 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10692 {
10693 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10694 HMVMX_RETURN_UNEXPECTED_EXIT();
10695 }
10696#endif
10697 break;
10698 }
10699 }
10700 }
10701#endif /* VBOX_STRICT */
10702 }
10703 return rc;
10704}
10705
10706
10707/**
10708 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10709 */
10710HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10711{
10712 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10713
10714 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10715 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10716 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10717 return VERR_EM_INTERPRETER;
10718 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10719 HMVMX_RETURN_UNEXPECTED_EXIT();
10720}
10721
10722
10723/**
10724 * VM-exit handler for when the TPR value is lowered below the specified
10725 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10726 */
10727HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10728{
10729 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10730 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10731
10732 /*
10733 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10734 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10735 * resume guest execution.
10736 */
10737 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10738 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10739 return VINF_SUCCESS;
10740}
10741
10742
10743/**
10744 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10745 * VM-exit.
10746 *
10747 * @retval VINF_SUCCESS when guest execution can continue.
10748 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10749 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10750 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10751 * recompiler.
10752 */
10753HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10754{
10755 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10756 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10757 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10758 AssertRCReturn(rc, rc);
10759
10760 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10761 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10762 PVM pVM = pVCpu->CTX_SUFF(pVM);
10763 switch (uAccessType)
10764 {
10765 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10766 {
10767#if 0
10768 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10769 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10770#else
10771 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10772 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10773 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10774#endif
10775 AssertRCReturn(rc, rc);
10776
10777 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10778 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10779 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10780 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10781
10782 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10783 {
10784 case 0: /* CR0 */
10785 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10786 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10787 break;
10788 case 2: /* CR2 */
10789 /* Nothing to do here, CR2 it's not part of the VMCS. */
10790 break;
10791 case 3: /* CR3 */
10792 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10793 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10794 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10795 break;
10796 case 4: /* CR4 */
10797 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10798 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10799 break;
10800 case 8: /* CR8 */
10801 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10802 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10803 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10804 break;
10805 default:
10806 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10807 break;
10808 }
10809
10810 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10811 break;
10812 }
10813
10814 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10815 {
10816 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10817 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10818 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10819 AssertRCReturn(rc, rc);
10820 Assert( !pVM->hm.s.fNestedPaging
10821 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10822 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10823
10824 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10825 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10826 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10827
10828 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10829 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10830 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10831 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10833 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10834 break;
10835 }
10836
10837 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10838 {
10839 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10840 AssertRCReturn(rc, rc);
10841 rc = EMInterpretCLTS(pVM, pVCpu);
10842 AssertRCReturn(rc, rc);
10843 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10844 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10845 Log4(("CRX CLTS write rc=%d\n", rc));
10846 break;
10847 }
10848
10849 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10850 {
10851 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10852 AssertRCReturn(rc, rc);
10853 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10854 if (RT_LIKELY(rc == VINF_SUCCESS))
10855 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10857 Log4(("CRX LMSW write rc=%d\n", rc));
10858 break;
10859 }
10860
10861 default:
10862 {
10863 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10864 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10865 }
10866 }
10867
10868 /* Validate possible error codes. */
10869 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10870 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10871 if (RT_SUCCESS(rc))
10872 {
10873 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10874 AssertRCReturn(rc2, rc2);
10875 }
10876
10877 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10878 return rc;
10879}
10880
10881
10882/**
10883 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10884 * VM-exit.
10885 */
10886HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10887{
10888 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10889 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10890
10891 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10892 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10893 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10894 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10895 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10896 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10897 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10898 AssertRCReturn(rc2, rc2);
10899
10900 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10901 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10902 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10903 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10904 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10905 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10906 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10907 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10908
10909 /* I/O operation lookup arrays. */
10910 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10911 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10912
10913 VBOXSTRICTRC rcStrict;
10914 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10915 uint32_t const cbInstr = pVmxTransient->cbInstr;
10916 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10917 PVM pVM = pVCpu->CTX_SUFF(pVM);
10918 if (fIOString)
10919 {
10920#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10921 /*
10922 * INS/OUTS - I/O String instruction.
10923 *
10924 * Use instruction-information if available, otherwise fall back on
10925 * interpreting the instruction.
10926 */
10927 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10928 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10929 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10930 {
10931 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10932 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10933 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10934 AssertRCReturn(rc2, rc2);
10935 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10936 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10937 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10938 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10939 if (fIOWrite)
10940 {
10941 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10942 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10943 }
10944 else
10945 {
10946 /*
10947 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10948 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10949 * See Intel Instruction spec. for "INS".
10950 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10951 */
10952 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10953 }
10954 }
10955 else
10956 {
10957 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10958 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10959 AssertRCReturn(rc2, rc2);
10960 rcStrict = IEMExecOne(pVCpu);
10961 }
10962 /** @todo IEM needs to be setting these flags somehow. */
10963 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10964 fUpdateRipAlready = true;
10965#else
10966 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10967 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10968 if (RT_SUCCESS(rcStrict))
10969 {
10970 if (fIOWrite)
10971 {
10972 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10973 (DISCPUMODE)pDis->uAddrMode, cbValue);
10974 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10975 }
10976 else
10977 {
10978 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10979 (DISCPUMODE)pDis->uAddrMode, cbValue);
10980 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10981 }
10982 }
10983 else
10984 {
10985 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10986 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10987 }
10988#endif
10989 }
10990 else
10991 {
10992 /*
10993 * IN/OUT - I/O instruction.
10994 */
10995 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10996 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
10997 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10998 if (fIOWrite)
10999 {
11000 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11001 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11002 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11004 }
11005 else
11006 {
11007 uint32_t u32Result = 0;
11008 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11009 if (IOM_SUCCESS(rcStrict))
11010 {
11011 /* Save result of I/O IN instr. in AL/AX/EAX. */
11012 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11013 }
11014 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11015 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11017 }
11018 }
11019
11020 if (IOM_SUCCESS(rcStrict))
11021 {
11022 if (!fUpdateRipAlready)
11023 {
11024 pMixedCtx->rip += cbInstr;
11025 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11026 }
11027
11028 /*
11029 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11030 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11031 */
11032 if (fIOString)
11033 {
11034 /** @todo Single-step for INS/OUTS with REP prefix? */
11035 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11036 }
11037 else if (fStepping)
11038 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11039
11040 /*
11041 * If any I/O breakpoints are armed, we need to check if one triggered
11042 * and take appropriate action.
11043 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11044 */
11045 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11046 AssertRCReturn(rc2, rc2);
11047
11048 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11049 * execution engines about whether hyper BPs and such are pending. */
11050 uint32_t const uDr7 = pMixedCtx->dr[7];
11051 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11052 && X86_DR7_ANY_RW_IO(uDr7)
11053 && (pMixedCtx->cr4 & X86_CR4_DE))
11054 || DBGFBpIsHwIoArmed(pVM)))
11055 {
11056 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11057
11058 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11059 VMMRZCallRing3Disable(pVCpu);
11060 HM_DISABLE_PREEMPT_IF_NEEDED();
11061
11062 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
11063
11064 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11065 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11066 {
11067 /* Raise #DB. */
11068 if (fIsGuestDbgActive)
11069 ASMSetDR6(pMixedCtx->dr[6]);
11070 if (pMixedCtx->dr[7] != uDr7)
11071 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11072
11073 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11074 }
11075 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11076 else if ( rcStrict2 != VINF_SUCCESS
11077 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11078 rcStrict = rcStrict2;
11079
11080 HM_RESTORE_PREEMPT_IF_NEEDED();
11081 VMMRZCallRing3Enable(pVCpu);
11082 }
11083 }
11084
11085#ifdef DEBUG
11086 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11087 Assert(!fIOWrite);
11088 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11089 Assert(fIOWrite);
11090 else
11091 {
11092 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11093 * statuses, that the VMM device and some others may return. See
11094 * IOM_SUCCESS() for guidance. */
11095 AssertMsg( RT_FAILURE(rcStrict)
11096 || rcStrict == VINF_SUCCESS
11097 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11098 || rcStrict == VINF_EM_DBG_BREAKPOINT
11099 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11100 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11101 }
11102#endif
11103
11104 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11105 return VBOXSTRICTRC_TODO(rcStrict);
11106}
11107
11108
11109/**
11110 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11111 * VM-exit.
11112 */
11113HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11114{
11115 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11116
11117 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11118 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11119 AssertRCReturn(rc, rc);
11120 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11121 {
11122 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11123 AssertRCReturn(rc, rc);
11124 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11125 {
11126 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11127
11128 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11129 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11130
11131 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11132 Assert(!pVCpu->hm.s.Event.fPending);
11133 pVCpu->hm.s.Event.fPending = true;
11134 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11135 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11136 AssertRCReturn(rc, rc);
11137 if (fErrorCodeValid)
11138 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11139 else
11140 pVCpu->hm.s.Event.u32ErrCode = 0;
11141 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11142 && uVector == X86_XCPT_PF)
11143 {
11144 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11145 }
11146
11147 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11148 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11149 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11150 }
11151 }
11152
11153 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11154 * emulation. */
11155 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11156 return VERR_EM_INTERPRETER;
11157}
11158
11159
11160/**
11161 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11162 */
11163HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11164{
11165 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11166 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11167 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11168 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11169 AssertRCReturn(rc, rc);
11170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11171 return VINF_EM_DBG_STEPPED;
11172}
11173
11174
11175/**
11176 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11177 */
11178HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11179{
11180 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11181
11182 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11183 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11184 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11185 return VINF_SUCCESS;
11186 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11187 return rc;
11188
11189#if 0
11190 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11191 * just sync the whole thing. */
11192 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11193#else
11194 /* Aggressive state sync. for now. */
11195 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11196 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11197 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11198#endif
11199 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11200 AssertRCReturn(rc, rc);
11201
11202 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11203 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11204 switch (uAccessType)
11205 {
11206 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11207 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11208 {
11209 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11210 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
11211 {
11212 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11213 }
11214
11215 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11216 GCPhys &= PAGE_BASE_GC_MASK;
11217 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11218 PVM pVM = pVCpu->CTX_SUFF(pVM);
11219 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11220 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11221
11222 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11223 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11224 CPUMCTX2CORE(pMixedCtx), GCPhys);
11225 rc = VBOXSTRICTRC_VAL(rc2);
11226 Log4(("ApicAccess rc=%d\n", rc));
11227 if ( rc == VINF_SUCCESS
11228 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11229 || rc == VERR_PAGE_NOT_PRESENT)
11230 {
11231 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11232 | HM_CHANGED_GUEST_RSP
11233 | HM_CHANGED_GUEST_RFLAGS
11234 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11235 rc = VINF_SUCCESS;
11236 }
11237 break;
11238 }
11239
11240 default:
11241 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11242 rc = VINF_EM_RAW_EMULATE_INSTR;
11243 break;
11244 }
11245
11246 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11247 return rc;
11248}
11249
11250
11251/**
11252 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11253 * VM-exit.
11254 */
11255HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11256{
11257 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11258
11259 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11260 if (pVmxTransient->fWasGuestDebugStateActive)
11261 {
11262 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11263 HMVMX_RETURN_UNEXPECTED_EXIT();
11264 }
11265
11266 int rc = VERR_INTERNAL_ERROR_5;
11267 if ( !DBGFIsStepping(pVCpu)
11268 && !pVCpu->hm.s.fSingleInstruction
11269 && !pVmxTransient->fWasHyperDebugStateActive)
11270 {
11271 /* Don't intercept MOV DRx and #DB any more. */
11272 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11273 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11274 AssertRCReturn(rc, rc);
11275
11276 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11277 {
11278#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11279 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11280 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11281 AssertRCReturn(rc, rc);
11282#endif
11283 }
11284
11285 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11286 VMMRZCallRing3Disable(pVCpu);
11287 HM_DISABLE_PREEMPT_IF_NEEDED();
11288
11289 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11290 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11291 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11292
11293 HM_RESTORE_PREEMPT_IF_NEEDED();
11294 VMMRZCallRing3Enable(pVCpu);
11295
11296#ifdef VBOX_WITH_STATISTICS
11297 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11298 AssertRCReturn(rc, rc);
11299 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11300 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11301 else
11302 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11303#endif
11304 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11305 return VINF_SUCCESS;
11306 }
11307
11308 /*
11309 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11310 * Update the segment registers and DR7 from the CPU.
11311 */
11312 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11313 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11314 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11315 AssertRCReturn(rc, rc);
11316 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11317
11318 PVM pVM = pVCpu->CTX_SUFF(pVM);
11319 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11320 {
11321 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11322 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11323 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11324 if (RT_SUCCESS(rc))
11325 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11326 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11327 }
11328 else
11329 {
11330 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11331 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11332 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11333 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11334 }
11335
11336 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11337 if (RT_SUCCESS(rc))
11338 {
11339 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11340 AssertRCReturn(rc2, rc2);
11341 }
11342 return rc;
11343}
11344
11345
11346/**
11347 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11348 * Conditional VM-exit.
11349 */
11350HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11351{
11352 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11353 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11354
11355 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11356 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11357 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11358 return VINF_SUCCESS;
11359 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11360 return rc;
11361
11362 RTGCPHYS GCPhys = 0;
11363 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11364
11365#if 0
11366 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11367#else
11368 /* Aggressive state sync. for now. */
11369 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11370 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11371 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11372#endif
11373 AssertRCReturn(rc, rc);
11374
11375 /*
11376 * If we succeed, resume guest execution.
11377 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11378 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11379 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11380 * weird case. See @bugref{6043}.
11381 */
11382 PVM pVM = pVCpu->CTX_SUFF(pVM);
11383 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11384 rc = VBOXSTRICTRC_VAL(rc2);
11385 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11386 if ( rc == VINF_SUCCESS
11387 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11388 || rc == VERR_PAGE_NOT_PRESENT)
11389 {
11390 /* Successfully handled MMIO operation. */
11391 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11392 | HM_CHANGED_GUEST_RSP
11393 | HM_CHANGED_GUEST_RFLAGS
11394 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11395 rc = VINF_SUCCESS;
11396 }
11397 return rc;
11398}
11399
11400
11401/**
11402 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11403 * VM-exit.
11404 */
11405HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11406{
11407 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11408 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11409
11410 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11411 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11412 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11413 return VINF_SUCCESS;
11414 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11415 return rc;
11416
11417 RTGCPHYS GCPhys = 0;
11418 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11419 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11420#if 0
11421 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11422#else
11423 /* Aggressive state sync. for now. */
11424 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11425 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11426 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11427#endif
11428 AssertRCReturn(rc, rc);
11429
11430 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11431 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11432
11433 RTGCUINT uErrorCode = 0;
11434 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11435 uErrorCode |= X86_TRAP_PF_ID;
11436 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11437 uErrorCode |= X86_TRAP_PF_RW;
11438 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11439 uErrorCode |= X86_TRAP_PF_P;
11440
11441 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11442
11443 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11444 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11445
11446 /* Handle the pagefault trap for the nested shadow table. */
11447 PVM pVM = pVCpu->CTX_SUFF(pVM);
11448 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11449 TRPMResetTrap(pVCpu);
11450
11451 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11452 if ( rc == VINF_SUCCESS
11453 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11454 || rc == VERR_PAGE_NOT_PRESENT)
11455 {
11456 /* Successfully synced our nested page tables. */
11457 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11458 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11459 | HM_CHANGED_GUEST_RSP
11460 | HM_CHANGED_GUEST_RFLAGS);
11461 return VINF_SUCCESS;
11462 }
11463
11464 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11465 return rc;
11466}
11467
11468/** @} */
11469
11470/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11471/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11472/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11473
11474/** @name VM-exit exception handlers.
11475 * @{
11476 */
11477
11478/**
11479 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11480 */
11481static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11482{
11483 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11484 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11485
11486 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11487 AssertRCReturn(rc, rc);
11488
11489 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11490 {
11491 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11492 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11493
11494 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11495 * provides VM-exit instruction length. If this causes problem later,
11496 * disassemble the instruction like it's done on AMD-V. */
11497 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11498 AssertRCReturn(rc2, rc2);
11499 return rc;
11500 }
11501
11502 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11503 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11504 return rc;
11505}
11506
11507
11508/**
11509 * VM-exit exception handler for #BP (Breakpoint exception).
11510 */
11511static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11512{
11513 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11514 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11515
11516 /** @todo Try optimize this by not saving the entire guest state unless
11517 * really needed. */
11518 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11519 AssertRCReturn(rc, rc);
11520
11521 PVM pVM = pVCpu->CTX_SUFF(pVM);
11522 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11523 if (rc == VINF_EM_RAW_GUEST_TRAP)
11524 {
11525 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11526 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11527 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11528 AssertRCReturn(rc, rc);
11529
11530 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11531 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11532 }
11533
11534 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11535 return rc;
11536}
11537
11538
11539/**
11540 * VM-exit exception handler for #DB (Debug exception).
11541 */
11542static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11543{
11544 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11545 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11546 Log6(("XcptDB\n"));
11547
11548 /*
11549 * Get the DR6-like values from the exit qualification and pass it to DBGF
11550 * for processing.
11551 */
11552 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11553 AssertRCReturn(rc, rc);
11554
11555 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11556 uint64_t uDR6 = X86_DR6_INIT_VAL;
11557 uDR6 |= ( pVmxTransient->uExitQualification
11558 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11559
11560 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11561 if (rc == VINF_EM_RAW_GUEST_TRAP)
11562 {
11563 /*
11564 * The exception was for the guest. Update DR6, DR7.GD and
11565 * IA32_DEBUGCTL.LBR before forwarding it.
11566 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11567 */
11568 VMMRZCallRing3Disable(pVCpu);
11569 HM_DISABLE_PREEMPT_IF_NEEDED();
11570
11571 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11572 pMixedCtx->dr[6] |= uDR6;
11573 if (CPUMIsGuestDebugStateActive(pVCpu))
11574 ASMSetDR6(pMixedCtx->dr[6]);
11575
11576 HM_RESTORE_PREEMPT_IF_NEEDED();
11577 VMMRZCallRing3Enable(pVCpu);
11578
11579 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11580 AssertRCReturn(rc, rc);
11581
11582 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11583 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11584
11585 /* Paranoia. */
11586 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11587 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11588
11589 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11590 AssertRCReturn(rc, rc);
11591
11592 /*
11593 * Raise #DB in the guest.
11594 *
11595 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11596 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11597 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11598 *
11599 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11600 */
11601 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11602 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11603 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11604 AssertRCReturn(rc, rc);
11605 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11606 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11607 return VINF_SUCCESS;
11608 }
11609
11610 /*
11611 * Not a guest trap, must be a hypervisor related debug event then.
11612 * Update DR6 in case someone is interested in it.
11613 */
11614 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11615 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11616 CPUMSetHyperDR6(pVCpu, uDR6);
11617
11618 return rc;
11619}
11620
11621
11622/**
11623 * VM-exit exception handler for #NM (Device-not-available exception: floating
11624 * point exception).
11625 */
11626static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11627{
11628 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11629
11630 /* We require CR0 and EFER. EFER is always up-to-date. */
11631 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11632 AssertRCReturn(rc, rc);
11633
11634 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11635 VMMRZCallRing3Disable(pVCpu);
11636 HM_DISABLE_PREEMPT_IF_NEEDED();
11637
11638 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11639 if (pVmxTransient->fWasGuestFPUStateActive)
11640 {
11641 rc = VINF_EM_RAW_GUEST_TRAP;
11642 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11643 }
11644 else
11645 {
11646#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11647 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11648#endif
11649 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11650 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11651 }
11652
11653 HM_RESTORE_PREEMPT_IF_NEEDED();
11654 VMMRZCallRing3Enable(pVCpu);
11655
11656 if (rc == VINF_SUCCESS)
11657 {
11658 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11659 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11660 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11661 pVCpu->hm.s.fUseGuestFpu = true;
11662 }
11663 else
11664 {
11665 /* Forward #NM to the guest. */
11666 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11667 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11668 AssertRCReturn(rc, rc);
11669 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11670 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11671 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11672 }
11673
11674 return VINF_SUCCESS;
11675}
11676
11677
11678/**
11679 * VM-exit exception handler for #GP (General-protection exception).
11680 *
11681 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11682 */
11683static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11684{
11685 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11686 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11687
11688 int rc = VERR_INTERNAL_ERROR_5;
11689 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11690 {
11691#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11692 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11693 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11694 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11695 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11696 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11697 AssertRCReturn(rc, rc);
11698 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11699 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11700 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11701 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11702 return rc;
11703#else
11704 /* We don't intercept #GP. */
11705 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11706 NOREF(pVmxTransient);
11707 return VERR_VMX_UNEXPECTED_EXCEPTION;
11708#endif
11709 }
11710
11711 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11712 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11713
11714 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11715 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11716 AssertRCReturn(rc, rc);
11717
11718 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11719 uint32_t cbOp = 0;
11720 PVM pVM = pVCpu->CTX_SUFF(pVM);
11721 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11722 if (RT_SUCCESS(rc))
11723 {
11724 rc = VINF_SUCCESS;
11725 Assert(cbOp == pDis->cbInstr);
11726 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11727 switch (pDis->pCurInstr->uOpcode)
11728 {
11729 case OP_CLI:
11730 {
11731 pMixedCtx->eflags.Bits.u1IF = 0;
11732 pMixedCtx->rip += pDis->cbInstr;
11733 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11734 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11735 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11736 break;
11737 }
11738
11739 case OP_STI:
11740 {
11741 pMixedCtx->eflags.Bits.u1IF = 1;
11742 pMixedCtx->rip += pDis->cbInstr;
11743 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11744 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11745 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11746 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11747 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11748 break;
11749 }
11750
11751 case OP_HLT:
11752 {
11753 rc = VINF_EM_HALT;
11754 pMixedCtx->rip += pDis->cbInstr;
11755 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11756 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11757 break;
11758 }
11759
11760 case OP_POPF:
11761 {
11762 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11763 uint32_t cbParm;
11764 uint32_t uMask;
11765 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11766 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11767 {
11768 cbParm = 4;
11769 uMask = 0xffffffff;
11770 }
11771 else
11772 {
11773 cbParm = 2;
11774 uMask = 0xffff;
11775 }
11776
11777 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11778 RTGCPTR GCPtrStack = 0;
11779 X86EFLAGS Eflags;
11780 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11781 &GCPtrStack);
11782 if (RT_SUCCESS(rc))
11783 {
11784 Assert(sizeof(Eflags.u32) >= cbParm);
11785 Eflags.u32 = 0;
11786 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11787 }
11788 if (RT_FAILURE(rc))
11789 {
11790 rc = VERR_EM_INTERPRETER;
11791 break;
11792 }
11793 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11794 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11795 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11796 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11797 pMixedCtx->esp += cbParm;
11798 pMixedCtx->esp &= uMask;
11799 pMixedCtx->rip += pDis->cbInstr;
11800 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11801 | HM_CHANGED_GUEST_RSP
11802 | HM_CHANGED_GUEST_RFLAGS);
11803 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11804 if (fStepping)
11805 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11806
11807 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11808 break;
11809 }
11810
11811 case OP_PUSHF:
11812 {
11813 uint32_t cbParm;
11814 uint32_t uMask;
11815 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11816 {
11817 cbParm = 4;
11818 uMask = 0xffffffff;
11819 }
11820 else
11821 {
11822 cbParm = 2;
11823 uMask = 0xffff;
11824 }
11825
11826 /* Get the stack pointer & push the contents of eflags onto the stack. */
11827 RTGCPTR GCPtrStack = 0;
11828 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11829 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11830 if (RT_FAILURE(rc))
11831 {
11832 rc = VERR_EM_INTERPRETER;
11833 break;
11834 }
11835 X86EFLAGS Eflags = pMixedCtx->eflags;
11836 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11837 Eflags.Bits.u1RF = 0;
11838 Eflags.Bits.u1VM = 0;
11839
11840 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11841 if (RT_FAILURE(rc))
11842 {
11843 rc = VERR_EM_INTERPRETER;
11844 break;
11845 }
11846 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11847 pMixedCtx->esp -= cbParm;
11848 pMixedCtx->esp &= uMask;
11849 pMixedCtx->rip += pDis->cbInstr;
11850 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11851 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11852 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11853 break;
11854 }
11855
11856 case OP_IRET:
11857 {
11858 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11859 * instruction reference. */
11860 RTGCPTR GCPtrStack = 0;
11861 uint32_t uMask = 0xffff;
11862 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11863 uint16_t aIretFrame[3];
11864 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11865 {
11866 rc = VERR_EM_INTERPRETER;
11867 break;
11868 }
11869 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11870 &GCPtrStack);
11871 if (RT_SUCCESS(rc))
11872 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11873 if (RT_FAILURE(rc))
11874 {
11875 rc = VERR_EM_INTERPRETER;
11876 break;
11877 }
11878 pMixedCtx->eip = 0;
11879 pMixedCtx->ip = aIretFrame[0];
11880 pMixedCtx->cs.Sel = aIretFrame[1];
11881 pMixedCtx->cs.ValidSel = aIretFrame[1];
11882 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11883 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11884 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11885 pMixedCtx->sp += sizeof(aIretFrame);
11886 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11887 | HM_CHANGED_GUEST_SEGMENT_REGS
11888 | HM_CHANGED_GUEST_RSP
11889 | HM_CHANGED_GUEST_RFLAGS);
11890 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11891 if (fStepping)
11892 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11893 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11894 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11895 break;
11896 }
11897
11898 case OP_INT:
11899 {
11900 uint16_t uVector = pDis->Param1.uValue & 0xff;
11901 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11902 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11903 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11904 break;
11905 }
11906
11907 case OP_INTO:
11908 {
11909 if (pMixedCtx->eflags.Bits.u1OF)
11910 {
11911 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11912 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11913 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11914 }
11915 break;
11916 }
11917
11918 default:
11919 {
11920 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11921 EMCODETYPE_SUPERVISOR);
11922 rc = VBOXSTRICTRC_VAL(rc2);
11923 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11924 /** @todo We have to set pending-debug exceptions here when the guest is
11925 * single-stepping depending on the instruction that was interpreted. */
11926 Log4(("#GP rc=%Rrc\n", rc));
11927 break;
11928 }
11929 }
11930 }
11931 else
11932 rc = VERR_EM_INTERPRETER;
11933
11934 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11935 ("#GP Unexpected rc=%Rrc\n", rc));
11936 return rc;
11937}
11938
11939
11940#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11941/**
11942 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11943 * the exception reported in the VMX transient structure back into the VM.
11944 *
11945 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11946 * up-to-date.
11947 */
11948static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11949{
11950 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11951
11952 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11953 hmR0VmxCheckExitDueToEventDelivery(). */
11954 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11955 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11956 AssertRCReturn(rc, rc);
11957 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11958
11959#ifdef DEBUG_ramshankar
11960 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11961 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11962 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11963#endif
11964
11965 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11966 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11967 return VINF_SUCCESS;
11968}
11969#endif
11970
11971
11972/**
11973 * VM-exit exception handler for #PF (Page-fault exception).
11974 */
11975static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11976{
11977 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11978 PVM pVM = pVCpu->CTX_SUFF(pVM);
11979 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11980 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11981 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11982 AssertRCReturn(rc, rc);
11983
11984#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11985 if (pVM->hm.s.fNestedPaging)
11986 {
11987 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11988 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11989 {
11990 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11991 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11992 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11993 }
11994 else
11995 {
11996 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11997 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11998 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11999 }
12000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12001 return rc;
12002 }
12003#else
12004 Assert(!pVM->hm.s.fNestedPaging);
12005 NOREF(pVM);
12006#endif
12007
12008 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12009 AssertRCReturn(rc, rc);
12010
12011 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12012 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12013
12014 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12015 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12016 (RTGCPTR)pVmxTransient->uExitQualification);
12017
12018 Log4(("#PF: rc=%Rrc\n", rc));
12019 if (rc == VINF_SUCCESS)
12020 {
12021 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12022 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12023 * memory? We don't update the whole state here... */
12024 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12025 | HM_CHANGED_GUEST_RSP
12026 | HM_CHANGED_GUEST_RFLAGS
12027 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12028 TRPMResetTrap(pVCpu);
12029 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12030 return rc;
12031 }
12032 else if (rc == VINF_EM_RAW_GUEST_TRAP)
12033 {
12034 if (!pVmxTransient->fVectoringPF)
12035 {
12036 /* It's a guest page fault and needs to be reflected to the guest. */
12037 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12038 TRPMResetTrap(pVCpu);
12039 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12040 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12041 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12042 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12043 }
12044 else
12045 {
12046 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12047 TRPMResetTrap(pVCpu);
12048 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12049 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12050 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12051 }
12052
12053 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12054 return VINF_SUCCESS;
12055 }
12056
12057 TRPMResetTrap(pVCpu);
12058 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12059 return rc;
12060}
12061
12062/** @} */
12063
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette