VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 60422

Last change on this file since 60422 was 60380, checked in by vboxsync, 9 years ago

VMM/HMVMXR0: Use %RGp for printing guest physical addresses and not %RGv.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 572.5 KB
Line 
1/* $Id: HMVMXR0.cpp 60380 2016-04-07 16:07:58Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#include "HMInternal.h"
38#include <VBox/vmm/vm.h>
39#include "HMVMXR0.h"
40#include "dtrace/VBoxVMM.h"
41
42#ifdef DEBUG_ramshankar
43# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
44# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_CHECK_GUEST_STATE
47# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
48# define HMVMX_ALWAYS_TRAP_PF
49# define HMVMX_ALWAYS_SWAP_FPU_STATE
50# define HMVMX_ALWAYS_FLUSH_TLB
51# define HMVMX_ALWAYS_SWAP_EFER
52#endif
53
54
55/*********************************************************************************************************************************
56* Defined Constants And Macros *
57*********************************************************************************************************************************/
58/** Use the function table. */
59#define HMVMX_USE_FUNCTION_TABLE
60
61/** Determine which tagged-TLB flush handler to use. */
62#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67/** @name Updated-guest-state flags.
68 * @{ */
69#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
70#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
71#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
72#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
73#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
74#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
75#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
76#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
77#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
78#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
79#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
80#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
81#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
82#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
83#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
84#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
85#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
86#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
87#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
88#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
89#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
90 | HMVMX_UPDATED_GUEST_RSP \
91 | HMVMX_UPDATED_GUEST_RFLAGS \
92 | HMVMX_UPDATED_GUEST_CR0 \
93 | HMVMX_UPDATED_GUEST_CR3 \
94 | HMVMX_UPDATED_GUEST_CR4 \
95 | HMVMX_UPDATED_GUEST_GDTR \
96 | HMVMX_UPDATED_GUEST_IDTR \
97 | HMVMX_UPDATED_GUEST_LDTR \
98 | HMVMX_UPDATED_GUEST_TR \
99 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
100 | HMVMX_UPDATED_GUEST_DEBUG \
101 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
102 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
104 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
105 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
106 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
107 | HMVMX_UPDATED_GUEST_INTR_STATE \
108 | HMVMX_UPDATED_GUEST_APIC_STATE)
109/** @} */
110
111/** @name
112 * Flags to skip redundant reads of some common VMCS fields that are not part of
113 * the guest-CPU state but are in the transient structure.
114 */
115#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
117#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
122/** @} */
123
124/** @name
125 * States of the VMCS.
126 *
127 * This does not reflect all possible VMCS states but currently only those
128 * needed for maintaining the VMCS consistently even when thread-context hooks
129 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
130 */
131#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
132#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
133#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
134/** @} */
135
136/**
137 * Exception bitmap mask for real-mode guests (real-on-v86).
138 *
139 * We need to intercept all exceptions manually except:
140 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
141 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
142 * due to bugs in Intel CPUs.
143 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
144 * support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/** Helper macro for VM-exit handlers called unexpectedly. */
187#define HMVMX_RETURN_UNEXPECTED_EXIT() \
188 do { \
189 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
190 return VERR_VMX_UNEXPECTED_EXIT; \
191 } while (0)
192
193
194/*********************************************************************************************************************************
195* Structures and Typedefs *
196*********************************************************************************************************************************/
197/**
198 * VMX transient state.
199 *
200 * A state structure for holding miscellaneous information across
201 * VMX non-root operation and restored after the transition.
202 */
203typedef struct VMXTRANSIENT
204{
205 /** The host's rflags/eflags. */
206 RTCCUINTREG fEFlags;
207#if HC_ARCH_BITS == 32
208 uint32_t u32Alignment0;
209#endif
210 /** The guest's TPR value used for TPR shadowing. */
211 uint8_t u8GuestTpr;
212 /** Alignment. */
213 uint8_t abAlignment0[7];
214
215 /** The basic VM-exit reason. */
216 uint16_t uExitReason;
217 /** Alignment. */
218 uint16_t u16Alignment0;
219 /** The VM-exit interruption error code. */
220 uint32_t uExitIntErrorCode;
221 /** The VM-exit exit code qualification. */
222 uint64_t uExitQualification;
223
224 /** The VM-exit interruption-information field. */
225 uint32_t uExitIntInfo;
226 /** The VM-exit instruction-length field. */
227 uint32_t cbInstr;
228 /** The VM-exit instruction-information field. */
229 union
230 {
231 /** Plain unsigned int representation. */
232 uint32_t u;
233 /** INS and OUTS information. */
234 struct
235 {
236 uint32_t u7Reserved0 : 7;
237 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
238 uint32_t u3AddrSize : 3;
239 uint32_t u5Reserved1 : 5;
240 /** The segment register (X86_SREG_XXX). */
241 uint32_t iSegReg : 3;
242 uint32_t uReserved2 : 14;
243 } StrIo;
244 } ExitInstrInfo;
245 /** Whether the VM-entry failed or not. */
246 bool fVMEntryFailed;
247 /** Alignment. */
248 uint8_t abAlignment1[3];
249
250 /** The VM-entry interruption-information field. */
251 uint32_t uEntryIntInfo;
252 /** The VM-entry exception error code field. */
253 uint32_t uEntryXcptErrorCode;
254 /** The VM-entry instruction length field. */
255 uint32_t cbEntryInstr;
256
257 /** IDT-vectoring information field. */
258 uint32_t uIdtVectoringInfo;
259 /** IDT-vectoring error code. */
260 uint32_t uIdtVectoringErrorCode;
261
262 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
263 uint32_t fVmcsFieldsRead;
264
265 /** Whether the guest FPU was active at the time of VM-exit. */
266 bool fWasGuestFPUStateActive;
267 /** Whether the guest debug state was active at the time of VM-exit. */
268 bool fWasGuestDebugStateActive;
269 /** Whether the hyper debug state was active at the time of VM-exit. */
270 bool fWasHyperDebugStateActive;
271 /** Whether TSC-offsetting should be setup before VM-entry. */
272 bool fUpdateTscOffsettingAndPreemptTimer;
273 /** Whether the VM-exit was caused by a page-fault during delivery of a
274 * contributory exception or a page-fault. */
275 bool fVectoringDoublePF;
276 /** Whether the VM-exit was caused by a page-fault during delivery of an
277 * external interrupt or NMI. */
278 bool fVectoringPF;
279} VMXTRANSIENT;
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
284AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
285/** Pointer to VMX transient state. */
286typedef VMXTRANSIENT *PVMXTRANSIENT;
287
288
289/**
290 * MSR-bitmap read permissions.
291 */
292typedef enum VMXMSREXITREAD
293{
294 /** Reading this MSR causes a VM-exit. */
295 VMXMSREXIT_INTERCEPT_READ = 0xb,
296 /** Reading this MSR does not cause a VM-exit. */
297 VMXMSREXIT_PASSTHRU_READ
298} VMXMSREXITREAD;
299/** Pointer to MSR-bitmap read permissions. */
300typedef VMXMSREXITREAD* PVMXMSREXITREAD;
301
302/**
303 * MSR-bitmap write permissions.
304 */
305typedef enum VMXMSREXITWRITE
306{
307 /** Writing to this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
309 /** Writing to this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_WRITE
311} VMXMSREXITWRITE;
312/** Pointer to MSR-bitmap write permissions. */
313typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
314
315
316/**
317 * VMX VM-exit handler.
318 *
319 * @returns Strict VBox status code (i.e. informational status codes too).
320 * @param pVCpu The cross context virtual CPU structure.
321 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
322 * out-of-sync. Make sure to update the required
323 * fields before using them.
324 * @param pVmxTransient Pointer to the VMX-transient structure.
325 */
326#ifndef HMVMX_USE_FUNCTION_TABLE
327typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
328#else
329typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330/** Pointer to VM-exit handler. */
331typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
332#endif
333
334/**
335 * VMX VM-exit handler, non-strict status code.
336 *
337 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
338 *
339 * @returns VBox status code, no informational status code returned.
340 * @param pVCpu The cross context virtual CPU structure.
341 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
342 * out-of-sync. Make sure to update the required
343 * fields before using them.
344 * @param pVmxTransient Pointer to the VMX-transient structure.
345 *
346 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
347 * use of that status code will be replaced with VINF_EM_SOMETHING
348 * later when switching over to IEM.
349 */
350#ifndef HMVMX_USE_FUNCTION_TABLE
351typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
352#else
353typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
354#endif
355
356
357/*********************************************************************************************************************************
358* Internal Functions *
359*********************************************************************************************************************************/
360static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
361static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
362static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
363static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
364 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
365 bool fStepping, uint32_t *puIntState);
366#if HC_ARCH_BITS == 32
367static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
368#endif
369#ifndef HMVMX_USE_FUNCTION_TABLE
370DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
371# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
372# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
373#else
374# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
376#endif
377
378
379/** @name VM-exit handlers.
380 * @{
381 */
382static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
383static FNVMXEXITHANDLER hmR0VmxExitExtInt;
384static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
385static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
391static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
392static FNVMXEXITHANDLER hmR0VmxExitCpuid;
393static FNVMXEXITHANDLER hmR0VmxExitGetsec;
394static FNVMXEXITHANDLER hmR0VmxExitHlt;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
396static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
397static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
398static FNVMXEXITHANDLER hmR0VmxExitVmcall;
399static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
402static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
403static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
404static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
405static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
406static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
410static FNVMXEXITHANDLER hmR0VmxExitMwait;
411static FNVMXEXITHANDLER hmR0VmxExitMtf;
412static FNVMXEXITHANDLER hmR0VmxExitMonitor;
413static FNVMXEXITHANDLER hmR0VmxExitPause;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
416static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
417static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
418static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
420static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
421static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
422static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
423static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
424static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
425static FNVMXEXITHANDLER hmR0VmxExitRdrand;
426static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
427/** @} */
428
429static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
430static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
438
439
440/*********************************************************************************************************************************
441* Global Variables *
442*********************************************************************************************************************************/
443#ifdef HMVMX_USE_FUNCTION_TABLE
444
445/**
446 * VMX_EXIT dispatch table.
447 */
448static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
449{
450 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
451 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
452 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
453 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
454 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
455 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
456 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
457 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
458 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
459 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
460 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
461 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
462 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
463 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
464 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
465 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
466 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
467 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
468 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
469 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
470 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
471 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
472 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
473 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
474 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
475 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
476 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
477 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
478 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
479 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
480 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
481 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
482 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
483 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
484 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
485 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
486 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
487 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
488 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
490 /* 40 UNDEFINED */ hmR0VmxExitPause,
491 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
492 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
493 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
494 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
495 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
496 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
497 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
498 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
499 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
500 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
501 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
502 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
503 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
504 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
505 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
506 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
507 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
508 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
509 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
510 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
511 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
512 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
513 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
514 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
515};
516#endif /* HMVMX_USE_FUNCTION_TABLE */
517
518#ifdef VBOX_STRICT
519static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
520{
521 /* 0 */ "(Not Used)",
522 /* 1 */ "VMCALL executed in VMX root operation.",
523 /* 2 */ "VMCLEAR with invalid physical address.",
524 /* 3 */ "VMCLEAR with VMXON pointer.",
525 /* 4 */ "VMLAUNCH with non-clear VMCS.",
526 /* 5 */ "VMRESUME with non-launched VMCS.",
527 /* 6 */ "VMRESUME after VMXOFF",
528 /* 7 */ "VM-entry with invalid control fields.",
529 /* 8 */ "VM-entry with invalid host state fields.",
530 /* 9 */ "VMPTRLD with invalid physical address.",
531 /* 10 */ "VMPTRLD with VMXON pointer.",
532 /* 11 */ "VMPTRLD with incorrect revision identifier.",
533 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
534 /* 13 */ "VMWRITE to read-only VMCS component.",
535 /* 14 */ "(Not Used)",
536 /* 15 */ "VMXON executed in VMX root operation.",
537 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
538 /* 17 */ "VM-entry with non-launched executing VMCS.",
539 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
540 /* 19 */ "VMCALL with non-clear VMCS.",
541 /* 20 */ "VMCALL with invalid VM-exit control fields.",
542 /* 21 */ "(Not Used)",
543 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
544 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
545 /* 24 */ "VMCALL with invalid SMM-monitor features.",
546 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
547 /* 26 */ "VM-entry with events blocked by MOV SS.",
548 /* 27 */ "(Not Used)",
549 /* 28 */ "Invalid operand to INVEPT/INVVPID."
550};
551#endif /* VBOX_STRICT */
552
553
554
555/**
556 * Updates the VM's last error record.
557 *
558 * If there was a VMX instruction error, reads the error data from the VMCS and
559 * updates VCPU's last error record as well.
560 *
561 * @param pVM The cross context VM structure.
562 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
563 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
564 * VERR_VMX_INVALID_VMCS_FIELD.
565 * @param rc The error code.
566 */
567static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
568{
569 AssertPtr(pVM);
570 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
571 || rc == VERR_VMX_UNABLE_TO_START_VM)
572 {
573 AssertPtrReturnVoid(pVCpu);
574 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
575 }
576 pVM->hm.s.lLastError = rc;
577}
578
579
580/**
581 * Reads the VM-entry interruption-information field from the VMCS into the VMX
582 * transient structure.
583 *
584 * @returns VBox status code.
585 * @param pVmxTransient Pointer to the VMX transient structure.
586 *
587 * @remarks No-long-jump zone!!!
588 */
589DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
590{
591 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
592 AssertRCReturn(rc, rc);
593 return VINF_SUCCESS;
594}
595
596
597/**
598 * Reads the VM-entry exception error code field from the VMCS into
599 * the VMX transient structure.
600 *
601 * @returns VBox status code.
602 * @param pVmxTransient Pointer to the VMX transient structure.
603 *
604 * @remarks No-long-jump zone!!!
605 */
606DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
607{
608 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
609 AssertRCReturn(rc, rc);
610 return VINF_SUCCESS;
611}
612
613
614/**
615 * Reads the VM-entry exception error code field from the VMCS into
616 * the VMX transient structure.
617 *
618 * @returns VBox status code.
619 * @param pVmxTransient Pointer to the VMX transient structure.
620 *
621 * @remarks No-long-jump zone!!!
622 */
623DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
624{
625 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
626 AssertRCReturn(rc, rc);
627 return VINF_SUCCESS;
628}
629
630
631/**
632 * Reads the VM-exit interruption-information field from the VMCS into the VMX
633 * transient structure.
634 *
635 * @returns VBox status code.
636 * @param pVmxTransient Pointer to the VMX transient structure.
637 */
638DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
639{
640 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
641 {
642 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
643 AssertRCReturn(rc, rc);
644 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
645 }
646 return VINF_SUCCESS;
647}
648
649
650/**
651 * Reads the VM-exit interruption error code from the VMCS into the VMX
652 * transient structure.
653 *
654 * @returns VBox status code.
655 * @param pVmxTransient Pointer to the VMX transient structure.
656 */
657DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
658{
659 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
660 {
661 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
662 AssertRCReturn(rc, rc);
663 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
664 }
665 return VINF_SUCCESS;
666}
667
668
669/**
670 * Reads the VM-exit instruction length field from the VMCS into the VMX
671 * transient structure.
672 *
673 * @returns VBox status code.
674 * @param pVmxTransient Pointer to the VMX transient structure.
675 */
676DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
677{
678 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
679 {
680 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
681 AssertRCReturn(rc, rc);
682 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
683 }
684 return VINF_SUCCESS;
685}
686
687
688/**
689 * Reads the VM-exit instruction-information field from the VMCS into
690 * the VMX transient structure.
691 *
692 * @returns VBox status code.
693 * @param pVmxTransient Pointer to the VMX transient structure.
694 */
695DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
696{
697 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
698 {
699 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
700 AssertRCReturn(rc, rc);
701 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
702 }
703 return VINF_SUCCESS;
704}
705
706
707/**
708 * Reads the exit code qualification from the VMCS into the VMX transient
709 * structure.
710 *
711 * @returns VBox status code.
712 * @param pVCpu The cross context virtual CPU structure of the
713 * calling EMT. (Required for the VMCS cache case.)
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 */
716DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
717{
718 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
719 {
720 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
721 AssertRCReturn(rc, rc);
722 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
723 }
724 return VINF_SUCCESS;
725}
726
727
728/**
729 * Reads the IDT-vectoring information field from the VMCS into the VMX
730 * transient structure.
731 *
732 * @returns VBox status code.
733 * @param pVmxTransient Pointer to the VMX transient structure.
734 *
735 * @remarks No-long-jump zone!!!
736 */
737DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
738{
739 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
740 {
741 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
742 AssertRCReturn(rc, rc);
743 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
744 }
745 return VINF_SUCCESS;
746}
747
748
749/**
750 * Reads the IDT-vectoring error code from the VMCS into the VMX
751 * transient structure.
752 *
753 * @returns VBox status code.
754 * @param pVmxTransient Pointer to the VMX transient structure.
755 */
756DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
757{
758 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
759 {
760 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
761 AssertRCReturn(rc, rc);
762 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
763 }
764 return VINF_SUCCESS;
765}
766
767
768/**
769 * Enters VMX root mode operation on the current CPU.
770 *
771 * @returns VBox status code.
772 * @param pVM The cross context VM structure. Can be
773 * NULL, after a resume.
774 * @param HCPhysCpuPage Physical address of the VMXON region.
775 * @param pvCpuPage Pointer to the VMXON region.
776 */
777static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
778{
779 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
780 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
781 Assert(pvCpuPage);
782 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
783
784 if (pVM)
785 {
786 /* Write the VMCS revision dword to the VMXON region. */
787 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
788 }
789
790 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
791 RTCCUINTREG fEFlags = ASMIntDisableFlags();
792
793 /* Enable the VMX bit in CR4 if necessary. */
794 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
795
796 /* Enter VMX root mode. */
797 int rc = VMXEnable(HCPhysCpuPage);
798 if (RT_FAILURE(rc))
799 {
800 if (!(uOldCr4 & X86_CR4_VMXE))
801 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
802
803 if (pVM)
804 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
805 }
806
807 /* Restore interrupts. */
808 ASMSetFlags(fEFlags);
809 return rc;
810}
811
812
813/**
814 * Exits VMX root mode operation on the current CPU.
815 *
816 * @returns VBox status code.
817 */
818static int hmR0VmxLeaveRootMode(void)
819{
820 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
821
822 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
823 RTCCUINTREG fEFlags = ASMIntDisableFlags();
824
825 /* If we're for some reason not in VMX root mode, then don't leave it. */
826 RTCCUINTREG uHostCR4 = ASMGetCR4();
827
828 int rc;
829 if (uHostCR4 & X86_CR4_VMXE)
830 {
831 /* Exit VMX root mode and clear the VMX bit in CR4. */
832 VMXDisable();
833 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
834 rc = VINF_SUCCESS;
835 }
836 else
837 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
838
839 /* Restore interrupts. */
840 ASMSetFlags(fEFlags);
841 return rc;
842}
843
844
845/**
846 * Allocates and maps one physically contiguous page. The allocated page is
847 * zero'd out. (Used by various VT-x structures).
848 *
849 * @returns IPRT status code.
850 * @param pMemObj Pointer to the ring-0 memory object.
851 * @param ppVirt Where to store the virtual address of the
852 * allocation.
853 * @param pHCPhys Where to store the physical address of the
854 * allocation.
855 */
856DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
857{
858 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
859 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
860 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
861
862 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
863 if (RT_FAILURE(rc))
864 return rc;
865 *ppVirt = RTR0MemObjAddress(*pMemObj);
866 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
867 ASMMemZero32(*ppVirt, PAGE_SIZE);
868 return VINF_SUCCESS;
869}
870
871
872/**
873 * Frees and unmaps an allocated physical page.
874 *
875 * @param pMemObj Pointer to the ring-0 memory object.
876 * @param ppVirt Where to re-initialize the virtual address of
877 * allocation as 0.
878 * @param pHCPhys Where to re-initialize the physical address of the
879 * allocation as 0.
880 */
881DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
882{
883 AssertPtr(pMemObj);
884 AssertPtr(ppVirt);
885 AssertPtr(pHCPhys);
886 if (*pMemObj != NIL_RTR0MEMOBJ)
887 {
888 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
889 AssertRC(rc);
890 *pMemObj = NIL_RTR0MEMOBJ;
891 *ppVirt = 0;
892 *pHCPhys = 0;
893 }
894}
895
896
897/**
898 * Worker function to free VT-x related structures.
899 *
900 * @returns IPRT status code.
901 * @param pVM The cross context VM structure.
902 */
903static void hmR0VmxStructsFree(PVM pVM)
904{
905 for (VMCPUID i = 0; i < pVM->cCpus; i++)
906 {
907 PVMCPU pVCpu = &pVM->aCpus[i];
908 AssertPtr(pVCpu);
909
910 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
911 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
912
913 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
915
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
918 }
919
920 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
921#ifdef VBOX_WITH_CRASHDUMP_MAGIC
922 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
923#endif
924}
925
926
927/**
928 * Worker function to allocate VT-x related VM structures.
929 *
930 * @returns IPRT status code.
931 * @param pVM The cross context VM structure.
932 */
933static int hmR0VmxStructsAlloc(PVM pVM)
934{
935 /*
936 * Initialize members up-front so we can cleanup properly on allocation failure.
937 */
938#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
939 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
940 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
941 pVM->hm.s.vmx.HCPhys##a_Name = 0;
942
943#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
944 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
947
948#ifdef VBOX_WITH_CRASHDUMP_MAGIC
949 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
950#endif
951 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
952
953 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
954 for (VMCPUID i = 0; i < pVM->cCpus; i++)
955 {
956 PVMCPU pVCpu = &pVM->aCpus[i];
957 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
958 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
959 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
962 }
963#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
964#undef VMXLOCAL_INIT_VM_MEMOBJ
965
966 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
967 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
968 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
969 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
970
971 /*
972 * Allocate all the VT-x structures.
973 */
974 int rc = VINF_SUCCESS;
975#ifdef VBOX_WITH_CRASHDUMP_MAGIC
976 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
977 if (RT_FAILURE(rc))
978 goto cleanup;
979 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
980 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
981#endif
982
983 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
984 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
985 {
986 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
987 &pVM->hm.s.vmx.HCPhysApicAccess);
988 if (RT_FAILURE(rc))
989 goto cleanup;
990 }
991
992 /*
993 * Initialize per-VCPU VT-x structures.
994 */
995 for (VMCPUID i = 0; i < pVM->cCpus; i++)
996 {
997 PVMCPU pVCpu = &pVM->aCpus[i];
998 AssertPtr(pVCpu);
999
1000 /* Allocate the VM control structure (VMCS). */
1001 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1002 if (RT_FAILURE(rc))
1003 goto cleanup;
1004
1005 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1006 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1007 {
1008 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1009 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1010 if (RT_FAILURE(rc))
1011 goto cleanup;
1012 }
1013
1014 /*
1015 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1016 * transparent accesses of specific MSRs.
1017 *
1018 * If the condition for enabling MSR bitmaps changes here, don't forget to
1019 * update HMAreMsrBitmapsAvailable().
1020 */
1021 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1022 {
1023 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1024 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1025 if (RT_FAILURE(rc))
1026 goto cleanup;
1027 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1028 }
1029
1030 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1031 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1032 if (RT_FAILURE(rc))
1033 goto cleanup;
1034
1035 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1036 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1037 if (RT_FAILURE(rc))
1038 goto cleanup;
1039 }
1040
1041 return VINF_SUCCESS;
1042
1043cleanup:
1044 hmR0VmxStructsFree(pVM);
1045 return rc;
1046}
1047
1048
1049/**
1050 * Does global VT-x initialization (called during module initialization).
1051 *
1052 * @returns VBox status code.
1053 */
1054VMMR0DECL(int) VMXR0GlobalInit(void)
1055{
1056#ifdef HMVMX_USE_FUNCTION_TABLE
1057 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1058# ifdef VBOX_STRICT
1059 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1060 Assert(g_apfnVMExitHandlers[i]);
1061# endif
1062#endif
1063 return VINF_SUCCESS;
1064}
1065
1066
1067/**
1068 * Does global VT-x termination (called during module termination).
1069 */
1070VMMR0DECL(void) VMXR0GlobalTerm()
1071{
1072 /* Nothing to do currently. */
1073}
1074
1075
1076/**
1077 * Sets up and activates VT-x on the current CPU.
1078 *
1079 * @returns VBox status code.
1080 * @param pCpu Pointer to the global CPU info struct.
1081 * @param pVM The cross context VM structure. Can be
1082 * NULL after a host resume operation.
1083 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1084 * fEnabledByHost is @c true).
1085 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1086 * @a fEnabledByHost is @c true).
1087 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1088 * enable VT-x on the host.
1089 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1090 */
1091VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1092 void *pvMsrs)
1093{
1094 Assert(pCpu);
1095 Assert(pvMsrs);
1096 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1097
1098 /* Enable VT-x if it's not already enabled by the host. */
1099 if (!fEnabledByHost)
1100 {
1101 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1102 if (RT_FAILURE(rc))
1103 return rc;
1104 }
1105
1106 /*
1107 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1108 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1109 */
1110 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1111 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1112 {
1113 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1114 pCpu->fFlushAsidBeforeUse = false;
1115 }
1116 else
1117 pCpu->fFlushAsidBeforeUse = true;
1118
1119 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1120 ++pCpu->cTlbFlushes;
1121
1122 return VINF_SUCCESS;
1123}
1124
1125
1126/**
1127 * Deactivates VT-x on the current CPU.
1128 *
1129 * @returns VBox status code.
1130 * @param pCpu Pointer to the global CPU info struct.
1131 * @param pvCpuPage Pointer to the VMXON region.
1132 * @param HCPhysCpuPage Physical address of the VMXON region.
1133 *
1134 * @remarks This function should never be called when SUPR0EnableVTx() or
1135 * similar was used to enable VT-x on the host.
1136 */
1137VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1138{
1139 NOREF(pCpu);
1140 NOREF(pvCpuPage);
1141 NOREF(HCPhysCpuPage);
1142
1143 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1144 return hmR0VmxLeaveRootMode();
1145}
1146
1147
1148/**
1149 * Sets the permission bits for the specified MSR in the MSR bitmap.
1150 *
1151 * @param pVCpu The cross context virtual CPU structure.
1152 * @param uMsr The MSR value.
1153 * @param enmRead Whether reading this MSR causes a VM-exit.
1154 * @param enmWrite Whether writing this MSR causes a VM-exit.
1155 */
1156static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1157{
1158 int32_t iBit;
1159 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1160
1161 /*
1162 * Layout:
1163 * 0x000 - 0x3ff - Low MSR read bits
1164 * 0x400 - 0x7ff - High MSR read bits
1165 * 0x800 - 0xbff - Low MSR write bits
1166 * 0xc00 - 0xfff - High MSR write bits
1167 */
1168 if (uMsr <= 0x00001FFF)
1169 iBit = uMsr;
1170 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1171 {
1172 iBit = uMsr - UINT32_C(0xC0000000);
1173 pbMsrBitmap += 0x400;
1174 }
1175 else
1176 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1177
1178 Assert(iBit <= 0x1fff);
1179 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1180 ASMBitSet(pbMsrBitmap, iBit);
1181 else
1182 ASMBitClear(pbMsrBitmap, iBit);
1183
1184 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1185 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1186 else
1187 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1188}
1189
1190
1191#ifdef VBOX_STRICT
1192/**
1193 * Gets the permission bits for the specified MSR in the MSR bitmap.
1194 *
1195 * @returns VBox status code.
1196 * @retval VINF_SUCCESS if the specified MSR is found.
1197 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1198 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1199 *
1200 * @param pVCpu The cross context virtual CPU structure.
1201 * @param uMsr The MSR.
1202 * @param penmRead Where to store the read permissions.
1203 * @param penmWrite Where to store the write permissions.
1204 */
1205static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1206{
1207 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1208 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1209 int32_t iBit;
1210 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1211
1212 /* See hmR0VmxSetMsrPermission() for the layout. */
1213 if (uMsr <= 0x00001FFF)
1214 iBit = uMsr;
1215 else if ( uMsr >= 0xC0000000
1216 && uMsr <= 0xC0001FFF)
1217 {
1218 iBit = (uMsr - 0xC0000000);
1219 pbMsrBitmap += 0x400;
1220 }
1221 else
1222 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1223
1224 Assert(iBit <= 0x1fff);
1225 if (ASMBitTest(pbMsrBitmap, iBit))
1226 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1227 else
1228 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1229
1230 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1231 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1232 else
1233 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1234 return VINF_SUCCESS;
1235}
1236#endif /* VBOX_STRICT */
1237
1238
1239/**
1240 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1241 * area.
1242 *
1243 * @returns VBox status code.
1244 * @param pVCpu The cross context virtual CPU structure.
1245 * @param cMsrs The number of MSRs.
1246 */
1247DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1248{
1249 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1250 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1251 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1252 {
1253 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1254 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1255 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1256 }
1257
1258 /* Update number of guest MSRs to load/store across the world-switch. */
1259 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1260 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1261
1262 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1264 AssertRCReturn(rc, rc);
1265
1266 /* Update the VCPU's copy of the MSR count. */
1267 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1268
1269 return VINF_SUCCESS;
1270}
1271
1272
1273/**
1274 * Adds a new (or updates the value of an existing) guest/host MSR
1275 * pair to be swapped during the world-switch as part of the
1276 * auto-load/store MSR area in the VMCS.
1277 *
1278 * @returns VBox status code.
1279 * @param pVCpu The cross context virtual CPU structure.
1280 * @param uMsr The MSR.
1281 * @param uGuestMsrValue Value of the guest MSR.
1282 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1283 * necessary.
1284 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1285 * its value was updated. Optional, can be NULL.
1286 */
1287static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1288 bool *pfAddedAndUpdated)
1289{
1290 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1291 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1292 uint32_t i;
1293 for (i = 0; i < cMsrs; i++)
1294 {
1295 if (pGuestMsr->u32Msr == uMsr)
1296 break;
1297 pGuestMsr++;
1298 }
1299
1300 bool fAdded = false;
1301 if (i == cMsrs)
1302 {
1303 ++cMsrs;
1304 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1305 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1306
1307 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1308 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1309 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1310
1311 fAdded = true;
1312 }
1313
1314 /* Update the MSR values in the auto-load/store MSR area. */
1315 pGuestMsr->u32Msr = uMsr;
1316 pGuestMsr->u64Value = uGuestMsrValue;
1317
1318 /* Create/update the MSR slot in the host MSR area. */
1319 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1320 pHostMsr += i;
1321 pHostMsr->u32Msr = uMsr;
1322
1323 /*
1324 * Update the host MSR only when requested by the caller AND when we're
1325 * adding it to the auto-load/store area. Otherwise, it would have been
1326 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1327 */
1328 bool fUpdatedMsrValue = false;
1329 if ( fAdded
1330 && fUpdateHostMsr)
1331 {
1332 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1333 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1334 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1335 fUpdatedMsrValue = true;
1336 }
1337
1338 if (pfAddedAndUpdated)
1339 *pfAddedAndUpdated = fUpdatedMsrValue;
1340 return VINF_SUCCESS;
1341}
1342
1343
1344/**
1345 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1346 * auto-load/store MSR area in the VMCS.
1347 *
1348 * @returns VBox status code.
1349 * @param pVCpu The cross context virtual CPU structure.
1350 * @param uMsr The MSR.
1351 */
1352static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1353{
1354 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1355 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1356 for (uint32_t i = 0; i < cMsrs; i++)
1357 {
1358 /* Find the MSR. */
1359 if (pGuestMsr->u32Msr == uMsr)
1360 {
1361 /* If it's the last MSR, simply reduce the count. */
1362 if (i == cMsrs - 1)
1363 {
1364 --cMsrs;
1365 break;
1366 }
1367
1368 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1369 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1370 pLastGuestMsr += cMsrs - 1;
1371 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1372 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1373
1374 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1375 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1376 pLastHostMsr += cMsrs - 1;
1377 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1378 pHostMsr->u64Value = pLastHostMsr->u64Value;
1379 --cMsrs;
1380 break;
1381 }
1382 pGuestMsr++;
1383 }
1384
1385 /* Update the VMCS if the count changed (meaning the MSR was found). */
1386 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1387 {
1388 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1389 AssertRCReturn(rc, rc);
1390
1391 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1392 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1393 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1394
1395 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1396 return VINF_SUCCESS;
1397 }
1398
1399 return VERR_NOT_FOUND;
1400}
1401
1402
1403/**
1404 * Checks if the specified guest MSR is part of the auto-load/store area in
1405 * the VMCS.
1406 *
1407 * @returns true if found, false otherwise.
1408 * @param pVCpu The cross context virtual CPU structure.
1409 * @param uMsr The MSR to find.
1410 */
1411static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1412{
1413 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1414 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1415
1416 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1417 {
1418 if (pGuestMsr->u32Msr == uMsr)
1419 return true;
1420 }
1421 return false;
1422}
1423
1424
1425/**
1426 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1427 *
1428 * @param pVCpu The cross context virtual CPU structure.
1429 *
1430 * @remarks No-long-jump zone!!!
1431 */
1432static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1433{
1434 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1435 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1436 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1437 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1438
1439 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1440 {
1441 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1442
1443 /*
1444 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1445 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1446 */
1447 if (pHostMsr->u32Msr == MSR_K6_EFER)
1448 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1449 else
1450 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1451 }
1452
1453 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1454}
1455
1456
1457#if HC_ARCH_BITS == 64
1458/**
1459 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1460 * perform lazy restoration of the host MSRs while leaving VT-x.
1461 *
1462 * @param pVCpu The cross context virtual CPU structure.
1463 *
1464 * @remarks No-long-jump zone!!!
1465 */
1466static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1467{
1468 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1469
1470 /*
1471 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1472 */
1473 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1474 {
1475 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1476 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1477 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1478 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1479 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1480 }
1481}
1482
1483
1484/**
1485 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1486 * lazily while leaving VT-x.
1487 *
1488 * @returns true if it does, false otherwise.
1489 * @param pVCpu The cross context virtual CPU structure.
1490 * @param uMsr The MSR to check.
1491 */
1492static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1493{
1494 NOREF(pVCpu);
1495 switch (uMsr)
1496 {
1497 case MSR_K8_LSTAR:
1498 case MSR_K6_STAR:
1499 case MSR_K8_SF_MASK:
1500 case MSR_K8_KERNEL_GS_BASE:
1501 return true;
1502 }
1503 return false;
1504}
1505
1506
1507/**
1508 * Saves a set of guest MSRs back into the guest-CPU context.
1509 *
1510 * @param pVCpu The cross context virtual CPU structure.
1511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1512 * out-of-sync. Make sure to update the required fields
1513 * before using them.
1514 *
1515 * @remarks No-long-jump zone!!!
1516 */
1517static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1518{
1519 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1520 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1521
1522 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1523 {
1524 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1525 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1526 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1527 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1528 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1529 }
1530}
1531
1532
1533/**
1534 * Loads a set of guests MSRs to allow read/passthru to the guest.
1535 *
1536 * The name of this function is slightly confusing. This function does NOT
1537 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1538 * common prefix for functions dealing with "lazy restoration" of the shared
1539 * MSRs.
1540 *
1541 * @param pVCpu The cross context virtual CPU structure.
1542 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1543 * out-of-sync. Make sure to update the required fields
1544 * before using them.
1545 *
1546 * @remarks No-long-jump zone!!!
1547 */
1548static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1549{
1550 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1551 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1552
1553#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1554 do { \
1555 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1556 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1557 else \
1558 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1559 } while (0)
1560
1561 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1562 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1563 {
1564 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1565 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1566 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1567 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1568 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1569 }
1570 else
1571 {
1572 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1573 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1574 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1575 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1576 }
1577
1578#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1579}
1580
1581
1582/**
1583 * Performs lazy restoration of the set of host MSRs if they were previously
1584 * loaded with guest MSR values.
1585 *
1586 * @param pVCpu The cross context virtual CPU structure.
1587 *
1588 * @remarks No-long-jump zone!!!
1589 * @remarks The guest MSRs should have been saved back into the guest-CPU
1590 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1591 */
1592static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1593{
1594 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1595 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1596
1597 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1598 {
1599 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1600 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1601 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1602 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1603 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1604 }
1605 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1606}
1607#endif /* HC_ARCH_BITS == 64 */
1608
1609
1610/**
1611 * Verifies that our cached values of the VMCS controls are all
1612 * consistent with what's actually present in the VMCS.
1613 *
1614 * @returns VBox status code.
1615 * @param pVCpu The cross context virtual CPU structure.
1616 */
1617static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1618{
1619 uint32_t u32Val;
1620 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1621 AssertRCReturn(rc, rc);
1622 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1623 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1624
1625 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1626 AssertRCReturn(rc, rc);
1627 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1628 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1629
1630 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1631 AssertRCReturn(rc, rc);
1632 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1633 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1634
1635 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1636 AssertRCReturn(rc, rc);
1637 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1638 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1639
1640 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1641 {
1642 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1643 AssertRCReturn(rc, rc);
1644 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1645 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1646 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1647 }
1648
1649 return VINF_SUCCESS;
1650}
1651
1652
1653#ifdef VBOX_STRICT
1654/**
1655 * Verifies that our cached host EFER value has not changed
1656 * since we cached it.
1657 *
1658 * @param pVCpu The cross context virtual CPU structure.
1659 */
1660static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1661{
1662 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1663
1664 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1665 {
1666 uint64_t u64Val;
1667 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1668 AssertRC(rc);
1669
1670 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1671 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1672 }
1673}
1674
1675
1676/**
1677 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1678 * VMCS are correct.
1679 *
1680 * @param pVCpu The cross context virtual CPU structure.
1681 */
1682static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1683{
1684 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1685
1686 /* Verify MSR counts in the VMCS are what we think it should be. */
1687 uint32_t cMsrs;
1688 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1689 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1690
1691 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1692 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1693
1694 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1695 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1696
1697 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1698 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1699 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1700 {
1701 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1702 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1703 pGuestMsr->u32Msr, cMsrs));
1704
1705 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1706 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1707 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1708
1709 /* Verify that the permissions are as expected in the MSR bitmap. */
1710 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1711 {
1712 VMXMSREXITREAD enmRead;
1713 VMXMSREXITWRITE enmWrite;
1714 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1715 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1716 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1717 {
1718 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1719 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1720 }
1721 else
1722 {
1723 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1724 pGuestMsr->u32Msr, cMsrs));
1725 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1726 pGuestMsr->u32Msr, cMsrs));
1727 }
1728 }
1729 }
1730}
1731#endif /* VBOX_STRICT */
1732
1733
1734/**
1735 * Flushes the TLB using EPT.
1736 *
1737 * @returns VBox status code.
1738 * @param pVCpu The cross context virtual CPU structure of the calling
1739 * EMT. Can be NULL depending on @a enmFlush.
1740 * @param enmFlush Type of flush.
1741 *
1742 * @remarks Caller is responsible for making sure this function is called only
1743 * when NestedPaging is supported and providing @a enmFlush that is
1744 * supported by the CPU.
1745 * @remarks Can be called with interrupts disabled.
1746 */
1747static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1748{
1749 uint64_t au64Descriptor[2];
1750 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1751 au64Descriptor[0] = 0;
1752 else
1753 {
1754 Assert(pVCpu);
1755 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1756 }
1757 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1758
1759 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1760 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1761 rc));
1762 if ( RT_SUCCESS(rc)
1763 && pVCpu)
1764 {
1765 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1766 }
1767}
1768
1769
1770/**
1771 * Flushes the TLB using VPID.
1772 *
1773 * @returns VBox status code.
1774 * @param pVM The cross context VM structure.
1775 * @param pVCpu The cross context virtual CPU structure of the calling
1776 * EMT. Can be NULL depending on @a enmFlush.
1777 * @param enmFlush Type of flush.
1778 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1779 * on @a enmFlush).
1780 *
1781 * @remarks Can be called with interrupts disabled.
1782 */
1783static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1784{
1785 NOREF(pVM);
1786 AssertPtr(pVM);
1787 Assert(pVM->hm.s.vmx.fVpid);
1788
1789 uint64_t au64Descriptor[2];
1790 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1791 {
1792 au64Descriptor[0] = 0;
1793 au64Descriptor[1] = 0;
1794 }
1795 else
1796 {
1797 AssertPtr(pVCpu);
1798 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1799 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1800 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1801 au64Descriptor[1] = GCPtr;
1802 }
1803
1804 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1805 AssertMsg(rc == VINF_SUCCESS,
1806 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1807 if ( RT_SUCCESS(rc)
1808 && pVCpu)
1809 {
1810 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1811 }
1812}
1813
1814
1815/**
1816 * Invalidates a guest page by guest virtual address. Only relevant for
1817 * EPT/VPID, otherwise there is nothing really to invalidate.
1818 *
1819 * @returns VBox status code.
1820 * @param pVM The cross context VM structure.
1821 * @param pVCpu The cross context virtual CPU structure.
1822 * @param GCVirt Guest virtual address of the page to invalidate.
1823 */
1824VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1825{
1826 AssertPtr(pVM);
1827 AssertPtr(pVCpu);
1828 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1829
1830 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1831 if (!fFlushPending)
1832 {
1833 /*
1834 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1835 * See @bugref{6043} and @bugref{6177}.
1836 *
1837 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1838 * function maybe called in a loop with individual addresses.
1839 */
1840 if (pVM->hm.s.vmx.fVpid)
1841 {
1842 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1843 {
1844 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1845 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1846 }
1847 else
1848 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1849 }
1850 else if (pVM->hm.s.fNestedPaging)
1851 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1852 }
1853
1854 return VINF_SUCCESS;
1855}
1856
1857
1858/**
1859 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1860 * otherwise there is nothing really to invalidate.
1861 *
1862 * @returns VBox status code.
1863 * @param pVM The cross context VM structure.
1864 * @param pVCpu The cross context virtual CPU structure.
1865 * @param GCPhys Guest physical address of the page to invalidate.
1866 */
1867VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1868{
1869 NOREF(pVM); NOREF(GCPhys);
1870 LogFlowFunc(("%RGp\n", GCPhys));
1871
1872 /*
1873 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1874 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1875 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1876 */
1877 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1878 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1879 return VINF_SUCCESS;
1880}
1881
1882
1883/**
1884 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1885 * case where neither EPT nor VPID is supported by the CPU.
1886 *
1887 * @param pVM The cross context VM structure.
1888 * @param pVCpu The cross context virtual CPU structure.
1889 * @param pCpu Pointer to the global HM struct.
1890 *
1891 * @remarks Called with interrupts disabled.
1892 */
1893static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1894{
1895 AssertPtr(pVCpu);
1896 AssertPtr(pCpu);
1897 NOREF(pVM);
1898
1899 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1900
1901 Assert(pCpu->idCpu != NIL_RTCPUID);
1902 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1903 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1904 pVCpu->hm.s.fForceTLBFlush = false;
1905 return;
1906}
1907
1908
1909/**
1910 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1911 *
1912 * @param pVM The cross context VM structure.
1913 * @param pVCpu The cross context virtual CPU structure.
1914 * @param pCpu Pointer to the global HM CPU struct.
1915 * @remarks All references to "ASID" in this function pertains to "VPID" in
1916 * Intel's nomenclature. The reason is, to avoid confusion in compare
1917 * statements since the host-CPU copies are named "ASID".
1918 *
1919 * @remarks Called with interrupts disabled.
1920 */
1921static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922{
1923#ifdef VBOX_WITH_STATISTICS
1924 bool fTlbFlushed = false;
1925# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1926# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1927 if (!fTlbFlushed) \
1928 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1929 } while (0)
1930#else
1931# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1932# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1933#endif
1934
1935 AssertPtr(pVM);
1936 AssertPtr(pCpu);
1937 AssertPtr(pVCpu);
1938 Assert(pCpu->idCpu != NIL_RTCPUID);
1939
1940 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1941 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1942 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1943
1944 /*
1945 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1946 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1947 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1948 */
1949 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1950 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1951 {
1952 ++pCpu->uCurrentAsid;
1953 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1954 {
1955 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1956 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1957 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1958 }
1959
1960 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1961 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1962 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1963
1964 /*
1965 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1966 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1967 */
1968 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1969 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1970 HMVMX_SET_TAGGED_TLB_FLUSHED();
1971 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1972 }
1973
1974 /* Check for explicit TLB flushes. */
1975 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1976 {
1977 /*
1978 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1979 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1980 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1981 * but not guest-physical mappings.
1982 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1983 */
1984 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1985 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1986 HMVMX_SET_TAGGED_TLB_FLUSHED();
1987 }
1988
1989 pVCpu->hm.s.fForceTLBFlush = false;
1990 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1991
1992 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1993 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1994 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1995 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1996 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1997 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1998 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1999 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2000 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2001
2002 /* Update VMCS with the VPID. */
2003 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2004 AssertRC(rc);
2005
2006#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2007}
2008
2009
2010/**
2011 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2012 *
2013 * @returns VBox status code.
2014 * @param pVM The cross context VM structure.
2015 * @param pVCpu The cross context virtual CPU structure.
2016 * @param pCpu Pointer to the global HM CPU struct.
2017 *
2018 * @remarks Called with interrupts disabled.
2019 */
2020static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2021{
2022 AssertPtr(pVM);
2023 AssertPtr(pVCpu);
2024 AssertPtr(pCpu);
2025 Assert(pCpu->idCpu != NIL_RTCPUID);
2026 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2027 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2028
2029 /*
2030 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2031 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2032 */
2033 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2034 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2035 {
2036 pVCpu->hm.s.fForceTLBFlush = true;
2037 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2038 }
2039
2040 /* Check for explicit TLB flushes. */
2041 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2042 {
2043 pVCpu->hm.s.fForceTLBFlush = true;
2044 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2045 }
2046
2047 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2048 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2049
2050 if (pVCpu->hm.s.fForceTLBFlush)
2051 {
2052 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2053 pVCpu->hm.s.fForceTLBFlush = false;
2054 }
2055}
2056
2057
2058/**
2059 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2060 *
2061 * @returns VBox status code.
2062 * @param pVM The cross context VM structure.
2063 * @param pVCpu The cross context virtual CPU structure.
2064 * @param pCpu Pointer to the global HM CPU struct.
2065 *
2066 * @remarks Called with interrupts disabled.
2067 */
2068static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2069{
2070 AssertPtr(pVM);
2071 AssertPtr(pVCpu);
2072 AssertPtr(pCpu);
2073 Assert(pCpu->idCpu != NIL_RTCPUID);
2074 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2075 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2076
2077 /*
2078 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2079 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2080 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2081 */
2082 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2083 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2084 {
2085 pVCpu->hm.s.fForceTLBFlush = true;
2086 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2087 }
2088
2089 /* Check for explicit TLB flushes. */
2090 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2091 {
2092 /*
2093 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2094 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2095 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2096 */
2097 pVCpu->hm.s.fForceTLBFlush = true;
2098 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2099 }
2100
2101 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2102 if (pVCpu->hm.s.fForceTLBFlush)
2103 {
2104 ++pCpu->uCurrentAsid;
2105 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2106 {
2107 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2108 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2109 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2110 }
2111
2112 pVCpu->hm.s.fForceTLBFlush = false;
2113 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2114 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2115 if (pCpu->fFlushAsidBeforeUse)
2116 {
2117 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2118 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2119 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2120 {
2121 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2122 pCpu->fFlushAsidBeforeUse = false;
2123 }
2124 else
2125 {
2126 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2127 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2128 }
2129 }
2130 }
2131
2132 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2133 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2134 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2135 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2136 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2137 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2138 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2139
2140 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2141 AssertRC(rc);
2142}
2143
2144
2145/**
2146 * Flushes the guest TLB entry based on CPU capabilities.
2147 *
2148 * @param pVCpu The cross context virtual CPU structure.
2149 * @param pCpu Pointer to the global HM CPU struct.
2150 */
2151DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2152{
2153#ifdef HMVMX_ALWAYS_FLUSH_TLB
2154 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2155#endif
2156 PVM pVM = pVCpu->CTX_SUFF(pVM);
2157 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2158 {
2159 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2160 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2161 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2162 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2163 default:
2164 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2165 break;
2166 }
2167
2168 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2169}
2170
2171
2172/**
2173 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2174 * TLB entries from the host TLB before VM-entry.
2175 *
2176 * @returns VBox status code.
2177 * @param pVM The cross context VM structure.
2178 */
2179static int hmR0VmxSetupTaggedTlb(PVM pVM)
2180{
2181 /*
2182 * Determine optimal flush type for Nested Paging.
2183 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2184 * guest execution (see hmR3InitFinalizeR0()).
2185 */
2186 if (pVM->hm.s.fNestedPaging)
2187 {
2188 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2189 {
2190 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2191 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2192 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2193 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2194 else
2195 {
2196 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2197 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2198 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2199 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2200 }
2201
2202 /* Make sure the write-back cacheable memory type for EPT is supported. */
2203 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2204 {
2205 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2206 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2207 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2208 }
2209
2210 /* EPT requires a page-walk length of 4. */
2211 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2212 {
2213 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2214 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2215 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2216 }
2217 }
2218 else
2219 {
2220 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2221 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2222 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2223 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2224 }
2225 }
2226
2227 /*
2228 * Determine optimal flush type for VPID.
2229 */
2230 if (pVM->hm.s.vmx.fVpid)
2231 {
2232 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2233 {
2234 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2235 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2236 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2237 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2238 else
2239 {
2240 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2241 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2242 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2243 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2244 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2245 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2246 pVM->hm.s.vmx.fVpid = false;
2247 }
2248 }
2249 else
2250 {
2251 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2252 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2253 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2254 pVM->hm.s.vmx.fVpid = false;
2255 }
2256 }
2257
2258 /*
2259 * Setup the handler for flushing tagged-TLBs.
2260 */
2261 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2262 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2263 else if (pVM->hm.s.fNestedPaging)
2264 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2265 else if (pVM->hm.s.vmx.fVpid)
2266 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2267 else
2268 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2269 return VINF_SUCCESS;
2270}
2271
2272
2273/**
2274 * Sets up pin-based VM-execution controls in the VMCS.
2275 *
2276 * @returns VBox status code.
2277 * @param pVM The cross context VM structure.
2278 * @param pVCpu The cross context virtual CPU structure.
2279 */
2280static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2281{
2282 AssertPtr(pVM);
2283 AssertPtr(pVCpu);
2284
2285 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2286 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2287
2288 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2289 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2290
2291 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2292 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2293
2294 /* Enable the VMX preemption timer. */
2295 if (pVM->hm.s.vmx.fUsePreemptTimer)
2296 {
2297 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2298 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2299 }
2300
2301#ifdef VBOX_WITH_NEW_APIC
2302#if 0
2303 /* Enable posted-interrupt processing. */
2304 if (pVM->hm.s.fPostedIntrs)
2305 {
2306 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2307 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2308 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2309 }
2310#endif
2311#endif
2312
2313 if ((val & zap) != val)
2314 {
2315 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2316 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2317 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2318 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2319 }
2320
2321 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2322 AssertRCReturn(rc, rc);
2323
2324 pVCpu->hm.s.vmx.u32PinCtls = val;
2325 return rc;
2326}
2327
2328
2329/**
2330 * Sets up processor-based VM-execution controls in the VMCS.
2331 *
2332 * @returns VBox status code.
2333 * @param pVM The cross context VM structure.
2334 * @param pVCpu The cross context virtual CPU structure.
2335 */
2336static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2337{
2338 AssertPtr(pVM);
2339 AssertPtr(pVCpu);
2340
2341 int rc = VERR_INTERNAL_ERROR_5;
2342 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2343 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2344
2345 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2346 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2347 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2348 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2349 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2350 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2351 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2352
2353 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2354 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2355 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2356 {
2357 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2358 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2359 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2360 }
2361
2362 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2363 if (!pVM->hm.s.fNestedPaging)
2364 {
2365 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2366 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2367 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2368 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2369 }
2370
2371 /* Use TPR shadowing if supported by the CPU. */
2372 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2373 {
2374 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2375 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2376 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2377 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2378 AssertRCReturn(rc, rc);
2379
2380 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2381 /* CR8 writes cause a VM-exit based on TPR threshold. */
2382 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2383 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2384 }
2385 else
2386 {
2387 /*
2388 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2389 * Set this control only for 64-bit guests.
2390 */
2391 if (pVM->hm.s.fAllow64BitGuests)
2392 {
2393 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2394 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2395 }
2396 }
2397
2398 /* Use MSR-bitmaps if supported by the CPU. */
2399 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2400 {
2401 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2402
2403 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2404 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2405 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2406 AssertRCReturn(rc, rc);
2407
2408 /*
2409 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2410 * automatically using dedicated fields in the VMCS.
2411 */
2412 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2413 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2414 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2415 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2416 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2417
2418#if HC_ARCH_BITS == 64
2419 /*
2420 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2421 */
2422 if (pVM->hm.s.fAllow64BitGuests)
2423 {
2424 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2425 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2426 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2427 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2428 }
2429#endif
2430 }
2431
2432 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2433 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2434 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2435
2436 if ((val & zap) != val)
2437 {
2438 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2439 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2440 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2441 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2442 }
2443
2444 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2445 AssertRCReturn(rc, rc);
2446
2447 pVCpu->hm.s.vmx.u32ProcCtls = val;
2448
2449 /*
2450 * Secondary processor-based VM-execution controls.
2451 */
2452 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2453 {
2454 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2455 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2456
2457 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2458 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2459
2460 if (pVM->hm.s.fNestedPaging)
2461 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2462 else
2463 {
2464 /*
2465 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2466 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2467 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2468 */
2469 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2470 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2471 }
2472
2473 if (pVM->hm.s.vmx.fVpid)
2474 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2475
2476 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2477 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2478
2479#ifdef VBOX_WITH_NEW_APIC
2480#if 0
2481 if (pVM->hm.s.fVirtApicRegs)
2482 {
2483 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2484 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2485
2486 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2487 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2488 }
2489#endif
2490#endif
2491
2492 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2493 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2494 * done dynamically. */
2495 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2496 {
2497 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2498 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2499 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2500 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2501 AssertRCReturn(rc, rc);
2502 }
2503
2504 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2505 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2506
2507 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2508 && pVM->hm.s.vmx.cPleGapTicks
2509 && pVM->hm.s.vmx.cPleWindowTicks)
2510 {
2511 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2512
2513 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2514 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2515 AssertRCReturn(rc, rc);
2516 }
2517
2518 if ((val & zap) != val)
2519 {
2520 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2521 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2522 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2523 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2524 }
2525
2526 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2527 AssertRCReturn(rc, rc);
2528
2529 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2530 }
2531 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2532 {
2533 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2534 "available\n"));
2535 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2536 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2537 }
2538
2539 return VINF_SUCCESS;
2540}
2541
2542
2543/**
2544 * Sets up miscellaneous (everything other than Pin & Processor-based
2545 * VM-execution) control fields in the VMCS.
2546 *
2547 * @returns VBox status code.
2548 * @param pVM The cross context VM structure.
2549 * @param pVCpu The cross context virtual CPU structure.
2550 */
2551static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2552{
2553 NOREF(pVM);
2554 AssertPtr(pVM);
2555 AssertPtr(pVCpu);
2556
2557 int rc = VERR_GENERAL_FAILURE;
2558
2559 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2560#if 0
2561 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2562 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2563 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2564
2565 /*
2566 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2567 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2568 * We thus use the exception bitmap to control it rather than use both.
2569 */
2570 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2571 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2572
2573 /** @todo Explore possibility of using IO-bitmaps. */
2574 /* All IO & IOIO instructions cause VM-exits. */
2575 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2576 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2577
2578 /* Initialize the MSR-bitmap area. */
2579 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2580 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2581 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2582 AssertRCReturn(rc, rc);
2583#endif
2584
2585 /* Setup MSR auto-load/store area. */
2586 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2587 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2588 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2589 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2590 AssertRCReturn(rc, rc);
2591
2592 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2593 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2594 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2595 AssertRCReturn(rc, rc);
2596
2597 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2598 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2599 AssertRCReturn(rc, rc);
2600
2601 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2602#if 0
2603 /* Setup debug controls */
2604 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2605 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2606 AssertRCReturn(rc, rc);
2607#endif
2608
2609 return rc;
2610}
2611
2612
2613/**
2614 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2615 *
2616 * @returns VBox status code.
2617 * @param pVM The cross context VM structure.
2618 * @param pVCpu The cross context virtual CPU structure.
2619 */
2620static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2621{
2622 AssertPtr(pVM);
2623 AssertPtr(pVCpu);
2624
2625 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2626
2627 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2628
2629 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2630 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2631
2632 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2633 and writes, and because recursive #DBs can cause the CPU hang, we must always
2634 intercept #DB. */
2635 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2636
2637 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2638 if (!pVM->hm.s.fNestedPaging)
2639 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2640
2641 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2642 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2643 AssertRCReturn(rc, rc);
2644 return rc;
2645}
2646
2647
2648/**
2649 * Sets up the initial guest-state mask. The guest-state mask is consulted
2650 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2651 * for the nested virtualization case (as it would cause a VM-exit).
2652 *
2653 * @param pVCpu The cross context virtual CPU structure.
2654 */
2655static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2656{
2657 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2658 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2659 return VINF_SUCCESS;
2660}
2661
2662
2663/**
2664 * Does per-VM VT-x initialization.
2665 *
2666 * @returns VBox status code.
2667 * @param pVM The cross context VM structure.
2668 */
2669VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2670{
2671 LogFlowFunc(("pVM=%p\n", pVM));
2672
2673 int rc = hmR0VmxStructsAlloc(pVM);
2674 if (RT_FAILURE(rc))
2675 {
2676 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2677 return rc;
2678 }
2679
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x termination.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2695 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2696 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2697#endif
2698 hmR0VmxStructsFree(pVM);
2699 return VINF_SUCCESS;
2700}
2701
2702
2703/**
2704 * Sets up the VM for execution under VT-x.
2705 * This function is only called once per-VM during initialization.
2706 *
2707 * @returns VBox status code.
2708 * @param pVM The cross context VM structure.
2709 */
2710VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2711{
2712 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2713 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2714
2715 LogFlowFunc(("pVM=%p\n", pVM));
2716
2717 /*
2718 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2719 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2720 */
2721 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2722 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2723 || !pVM->hm.s.vmx.pRealModeTSS))
2724 {
2725 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2726 return VERR_INTERNAL_ERROR;
2727 }
2728
2729 /* Initialize these always, see hmR3InitFinalizeR0().*/
2730 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2731 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2732
2733 /* Setup the tagged-TLB flush handlers. */
2734 int rc = hmR0VmxSetupTaggedTlb(pVM);
2735 if (RT_FAILURE(rc))
2736 {
2737 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2738 return rc;
2739 }
2740
2741 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2742 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2743#if HC_ARCH_BITS == 64
2744 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2745 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2746 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2747 {
2748 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2749 }
2750#endif
2751
2752 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2753 RTCCUINTREG uHostCR4 = ASMGetCR4();
2754 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2755 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2756
2757 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2758 {
2759 PVMCPU pVCpu = &pVM->aCpus[i];
2760 AssertPtr(pVCpu);
2761 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2762
2763 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2764 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2765
2766 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2767 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2768 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2769
2770 /* Set revision dword at the beginning of the VMCS structure. */
2771 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2772
2773 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2774 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2775 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2776 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2777
2778 /* Load this VMCS as the current VMCS. */
2779 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2780 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2781 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2782
2783 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2784 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2785 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2786
2787 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2788 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2789 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2790
2791 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2793 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2794
2795 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2801 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2802
2803#if HC_ARCH_BITS == 32
2804 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807#endif
2808
2809 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2810 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2815
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2817 }
2818
2819 return VINF_SUCCESS;
2820}
2821
2822
2823/**
2824 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2825 * the VMCS.
2826 *
2827 * @returns VBox status code.
2828 * @param pVM The cross context VM structure.
2829 * @param pVCpu The cross context virtual CPU structure.
2830 */
2831DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2832{
2833 NOREF(pVM); NOREF(pVCpu);
2834
2835 RTCCUINTREG uReg = ASMGetCR0();
2836 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2837 AssertRCReturn(rc, rc);
2838
2839 uReg = ASMGetCR3();
2840 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2841 AssertRCReturn(rc, rc);
2842
2843 uReg = ASMGetCR4();
2844 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2845 AssertRCReturn(rc, rc);
2846 return rc;
2847}
2848
2849
2850#if HC_ARCH_BITS == 64
2851/**
2852 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2853 * requirements. See hmR0VmxSaveHostSegmentRegs().
2854 */
2855# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2856 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2857 { \
2858 bool fValidSelector = true; \
2859 if ((selValue) & X86_SEL_LDT) \
2860 { \
2861 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2862 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2863 } \
2864 if (fValidSelector) \
2865 { \
2866 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2867 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2868 } \
2869 (selValue) = 0; \
2870 }
2871#endif
2872
2873
2874/**
2875 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2876 * the host-state area in the VMCS.
2877 *
2878 * @returns VBox status code.
2879 * @param pVM The cross context VM structure.
2880 * @param pVCpu The cross context virtual CPU structure.
2881 */
2882DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2883{
2884 int rc = VERR_INTERNAL_ERROR_5;
2885
2886#if HC_ARCH_BITS == 64
2887 /*
2888 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2889 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2890 */
2891 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2892 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2893#endif
2894
2895 /*
2896 * Host DS, ES, FS and GS segment registers.
2897 */
2898#if HC_ARCH_BITS == 64
2899 RTSEL uSelDS = ASMGetDS();
2900 RTSEL uSelES = ASMGetES();
2901 RTSEL uSelFS = ASMGetFS();
2902 RTSEL uSelGS = ASMGetGS();
2903#else
2904 RTSEL uSelDS = 0;
2905 RTSEL uSelES = 0;
2906 RTSEL uSelFS = 0;
2907 RTSEL uSelGS = 0;
2908#endif
2909
2910 /* Recalculate which host-state bits need to be manually restored. */
2911 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2912
2913 /*
2914 * Host CS and SS segment registers.
2915 */
2916 RTSEL uSelCS = ASMGetCS();
2917 RTSEL uSelSS = ASMGetSS();
2918
2919 /*
2920 * Host TR segment register.
2921 */
2922 RTSEL uSelTR = ASMGetTR();
2923
2924#if HC_ARCH_BITS == 64
2925 /*
2926 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2927 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2928 */
2929 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2930 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2931 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2932 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2933# undef VMXLOCAL_ADJUST_HOST_SEG
2934#endif
2935
2936 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2937 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2938 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2939 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2940 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2941 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2942 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2943 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2944 Assert(uSelCS);
2945 Assert(uSelTR);
2946
2947 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2948#if 0
2949 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2950 Assert(uSelSS != 0);
2951#endif
2952
2953 /* Write these host selector fields into the host-state area in the VMCS. */
2954 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2955 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2956#if HC_ARCH_BITS == 64
2957 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2958 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2959 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2960 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2961#else
2962 NOREF(uSelDS);
2963 NOREF(uSelES);
2964 NOREF(uSelFS);
2965 NOREF(uSelGS);
2966#endif
2967 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2968 AssertRCReturn(rc, rc);
2969
2970 /*
2971 * Host GDTR and IDTR.
2972 */
2973 RTGDTR Gdtr;
2974 RTIDTR Idtr;
2975 RT_ZERO(Gdtr);
2976 RT_ZERO(Idtr);
2977 ASMGetGDTR(&Gdtr);
2978 ASMGetIDTR(&Idtr);
2979 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2980 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2981 AssertRCReturn(rc, rc);
2982
2983#if HC_ARCH_BITS == 64
2984 /*
2985 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2986 * maximum limit (0xffff) on every VM-exit.
2987 */
2988 if (Gdtr.cbGdt != 0xffff)
2989 {
2990 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2991 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2992 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2993 }
2994
2995 /*
2996 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2997 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2998 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2999 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3000 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3001 * hosts where we are pretty sure it won't cause trouble.
3002 */
3003# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3004 if (Idtr.cbIdt < 0x0fff)
3005# else
3006 if (Idtr.cbIdt != 0xffff)
3007# endif
3008 {
3009 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3010 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3011 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3012 }
3013#endif
3014
3015 /*
3016 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3017 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3018 */
3019 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3020 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3021 VERR_VMX_INVALID_HOST_STATE);
3022
3023 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3024#if HC_ARCH_BITS == 64
3025 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3026
3027 /*
3028 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3029 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3030 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3031 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3032 *
3033 * [1] See Intel spec. 3.5 "System Descriptor Types".
3034 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3035 */
3036 Assert(pDesc->System.u4Type == 11);
3037 if ( pDesc->System.u16LimitLow != 0x67
3038 || pDesc->System.u4LimitHigh)
3039 {
3040 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3041 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3042 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3043 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3044 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3045
3046 /* Store the GDTR here as we need it while restoring TR. */
3047 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3048 }
3049#else
3050 NOREF(pVM);
3051 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3052#endif
3053 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3054 AssertRCReturn(rc, rc);
3055
3056 /*
3057 * Host FS base and GS base.
3058 */
3059#if HC_ARCH_BITS == 64
3060 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3061 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3062 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3063 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3064 AssertRCReturn(rc, rc);
3065
3066 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3067 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3068 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3069 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3070 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3071#endif
3072 return rc;
3073}
3074
3075
3076/**
3077 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3078 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3079 * the host after every successful VM-exit.
3080 *
3081 * @returns VBox status code.
3082 * @param pVM The cross context VM structure.
3083 * @param pVCpu The cross context virtual CPU structure.
3084 *
3085 * @remarks No-long-jump zone!!!
3086 */
3087DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3088{
3089 NOREF(pVM);
3090
3091 AssertPtr(pVCpu);
3092 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3093
3094 int rc = VINF_SUCCESS;
3095#if HC_ARCH_BITS == 64
3096 if (pVM->hm.s.fAllow64BitGuests)
3097 hmR0VmxLazySaveHostMsrs(pVCpu);
3098#endif
3099
3100 /*
3101 * Host Sysenter MSRs.
3102 */
3103 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3104#if HC_ARCH_BITS == 32
3105 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3106 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3107#else
3108 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3109 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3110#endif
3111 AssertRCReturn(rc, rc);
3112
3113 /*
3114 * Host EFER MSR.
3115 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3116 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3117 */
3118 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3119 {
3120 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3121 AssertRCReturn(rc, rc);
3122 }
3123
3124 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3125 * hmR0VmxLoadGuestExitCtls() !! */
3126
3127 return rc;
3128}
3129
3130
3131/**
3132 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3133 *
3134 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3135 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3136 * hmR0VMxLoadGuestEntryCtls().
3137 *
3138 * @returns true if we need to load guest EFER, false otherwise.
3139 * @param pVCpu The cross context virtual CPU structure.
3140 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3141 * out-of-sync. Make sure to update the required fields
3142 * before using them.
3143 *
3144 * @remarks Requires EFER, CR4.
3145 * @remarks No-long-jump zone!!!
3146 */
3147static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3148{
3149#ifdef HMVMX_ALWAYS_SWAP_EFER
3150 return true;
3151#endif
3152
3153#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3154 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3155 if (CPUMIsGuestInLongMode(pVCpu))
3156 return false;
3157#endif
3158
3159 PVM pVM = pVCpu->CTX_SUFF(pVM);
3160 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3161 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3162
3163 /*
3164 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3165 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3166 */
3167 if ( CPUMIsGuestInLongMode(pVCpu)
3168 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3169 {
3170 return true;
3171 }
3172
3173 /*
3174 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3175 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3176 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3177 */
3178 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3179 && (pMixedCtx->cr0 & X86_CR0_PG)
3180 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3181 {
3182 /* Assert that host is PAE capable. */
3183 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3184 return true;
3185 }
3186
3187 /** @todo Check the latest Intel spec. for any other bits,
3188 * like SMEP/SMAP? */
3189 return false;
3190}
3191
3192
3193/**
3194 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3195 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3196 * controls".
3197 *
3198 * @returns VBox status code.
3199 * @param pVCpu The cross context virtual CPU structure.
3200 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3201 * out-of-sync. Make sure to update the required fields
3202 * before using them.
3203 *
3204 * @remarks Requires EFER.
3205 * @remarks No-long-jump zone!!!
3206 */
3207DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3208{
3209 int rc = VINF_SUCCESS;
3210 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3211 {
3212 PVM pVM = pVCpu->CTX_SUFF(pVM);
3213 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3214 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3215
3216 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3217 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3218
3219 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3220 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3221 {
3222 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3223 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3224 }
3225 else
3226 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3227
3228 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3229 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3230 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3231 {
3232 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3233 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3234 }
3235
3236 /*
3237 * The following should -not- be set (since we're not in SMM mode):
3238 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3239 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3240 */
3241
3242 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3243 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3244
3245 if ((val & zap) != val)
3246 {
3247 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3248 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3249 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3250 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3251 }
3252
3253 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3254 AssertRCReturn(rc, rc);
3255
3256 pVCpu->hm.s.vmx.u32EntryCtls = val;
3257 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3258 }
3259 return rc;
3260}
3261
3262
3263/**
3264 * Sets up the VM-exit controls in the VMCS.
3265 *
3266 * @returns VBox status code.
3267 * @param pVCpu The cross context virtual CPU structure.
3268 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3269 * out-of-sync. Make sure to update the required fields
3270 * before using them.
3271 *
3272 * @remarks Requires EFER.
3273 */
3274DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3275{
3276 NOREF(pMixedCtx);
3277
3278 int rc = VINF_SUCCESS;
3279 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3280 {
3281 PVM pVM = pVCpu->CTX_SUFF(pVM);
3282 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3283 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3284
3285 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3286 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3287
3288 /*
3289 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3290 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3291 */
3292#if HC_ARCH_BITS == 64
3293 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3294 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3295#else
3296 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3297 {
3298 /* The switcher returns to long mode, EFER is managed by the switcher. */
3299 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3300 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3301 }
3302 else
3303 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3304#endif
3305
3306 /* If the newer VMCS fields for managing EFER exists, use it. */
3307 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3308 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3309 {
3310 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3311 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3312 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3313 }
3314
3315 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3316 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3317
3318 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3319 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3320 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3321
3322 if ( pVM->hm.s.vmx.fUsePreemptTimer
3323 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3324 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3325
3326 if ((val & zap) != val)
3327 {
3328 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3329 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3330 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3331 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3332 }
3333
3334 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3335 AssertRCReturn(rc, rc);
3336
3337 pVCpu->hm.s.vmx.u32ExitCtls = val;
3338 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3339 }
3340 return rc;
3341}
3342
3343
3344/**
3345 * Loads the guest APIC and related state.
3346 *
3347 * @returns VBox status code.
3348 * @param pVCpu The cross context virtual CPU structure.
3349 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3350 * out-of-sync. Make sure to update the required fields
3351 * before using them.
3352 */
3353DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3354{
3355 NOREF(pMixedCtx);
3356
3357 int rc = VINF_SUCCESS;
3358 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3359 {
3360 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3361 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3362 {
3363 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3364
3365 bool fPendingIntr = false;
3366 uint8_t u8Tpr = 0;
3367 uint8_t u8PendingIntr = 0;
3368 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3369 AssertRCReturn(rc, rc);
3370
3371 /*
3372 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3373 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3374 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3375 * the interrupt when we VM-exit for other reasons.
3376 */
3377 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3378 uint32_t u32TprThreshold = 0;
3379 if (fPendingIntr)
3380 {
3381 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3382 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3383 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3384 if (u8PendingPriority <= u8TprPriority)
3385 u32TprThreshold = u8PendingPriority;
3386 else
3387 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3388 }
3389 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3390
3391 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3392 AssertRCReturn(rc, rc);
3393 }
3394
3395 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3396 }
3397 return rc;
3398}
3399
3400
3401/**
3402 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3403 *
3404 * @returns Guest's interruptibility-state.
3405 * @param pVCpu The cross context virtual CPU structure.
3406 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3407 * out-of-sync. Make sure to update the required fields
3408 * before using them.
3409 *
3410 * @remarks No-long-jump zone!!!
3411 */
3412DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3413{
3414 /*
3415 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3416 */
3417 uint32_t uIntrState = 0;
3418 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3419 {
3420 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3421 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3422 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3423 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3424 {
3425 if (pMixedCtx->eflags.Bits.u1IF)
3426 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3427 else
3428 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3429 }
3430 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3431 }
3432
3433 /*
3434 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3435 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3436 * setting this would block host-NMIs and IRET will not clear the blocking.
3437 *
3438 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3439 */
3440 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3441 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3442 {
3443 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3444 }
3445
3446 return uIntrState;
3447}
3448
3449
3450/**
3451 * Loads the guest's interruptibility-state into the guest-state area in the
3452 * VMCS.
3453 *
3454 * @returns VBox status code.
3455 * @param pVCpu The cross context virtual CPU structure.
3456 * @param uIntrState The interruptibility-state to set.
3457 */
3458static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3459{
3460 NOREF(pVCpu);
3461 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3462 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3463 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3464 AssertRC(rc);
3465 return rc;
3466}
3467
3468
3469/**
3470 * Loads the exception intercepts required for guest execution in the VMCS.
3471 *
3472 * @returns VBox status code.
3473 * @param pVCpu The cross context virtual CPU structure.
3474 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3475 * out-of-sync. Make sure to update the required fields
3476 * before using them.
3477 */
3478static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3479{
3480 NOREF(pMixedCtx);
3481 int rc = VINF_SUCCESS;
3482 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3483 {
3484 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3485 if (pVCpu->hm.s.fGIMTrapXcptUD)
3486 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3487#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3488 else
3489 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3490#endif
3491
3492 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3493 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3494
3495 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3496 AssertRCReturn(rc, rc);
3497
3498 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3499 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3500 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3501 }
3502 return rc;
3503}
3504
3505
3506/**
3507 * Loads the guest's RIP into the guest-state area in the VMCS.
3508 *
3509 * @returns VBox status code.
3510 * @param pVCpu The cross context virtual CPU structure.
3511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3512 * out-of-sync. Make sure to update the required fields
3513 * before using them.
3514 *
3515 * @remarks No-long-jump zone!!!
3516 */
3517static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3518{
3519 int rc = VINF_SUCCESS;
3520 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3521 {
3522 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3523 AssertRCReturn(rc, rc);
3524
3525 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3526 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3527 HMCPU_CF_VALUE(pVCpu)));
3528 }
3529 return rc;
3530}
3531
3532
3533/**
3534 * Loads the guest's RSP into the guest-state area in the VMCS.
3535 *
3536 * @returns VBox status code.
3537 * @param pVCpu The cross context virtual CPU structure.
3538 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3539 * out-of-sync. Make sure to update the required fields
3540 * before using them.
3541 *
3542 * @remarks No-long-jump zone!!!
3543 */
3544static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3545{
3546 int rc = VINF_SUCCESS;
3547 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3548 {
3549 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3550 AssertRCReturn(rc, rc);
3551
3552 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3553 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3554 }
3555 return rc;
3556}
3557
3558
3559/**
3560 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3561 *
3562 * @returns VBox status code.
3563 * @param pVCpu The cross context virtual CPU structure.
3564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3565 * out-of-sync. Make sure to update the required fields
3566 * before using them.
3567 *
3568 * @remarks No-long-jump zone!!!
3569 */
3570static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3571{
3572 int rc = VINF_SUCCESS;
3573 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3574 {
3575 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3576 Let us assert it as such and use 32-bit VMWRITE. */
3577 Assert(!(pMixedCtx->rflags.u64 >> 32));
3578 X86EFLAGS Eflags = pMixedCtx->eflags;
3579 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3580 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3581 * These will never be cleared/set, unless some other part of the VMM
3582 * code is buggy - in which case we're better of finding and fixing
3583 * those bugs than hiding them. */
3584 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3585 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3586 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3587 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3588
3589 /*
3590 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3591 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3592 */
3593 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3594 {
3595 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3596 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3597 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3598 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3599 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3600 }
3601
3602 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3603 AssertRCReturn(rc, rc);
3604
3605 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3606 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3607 }
3608 return rc;
3609}
3610
3611
3612/**
3613 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3614 *
3615 * @returns VBox status code.
3616 * @param pVCpu The cross context virtual CPU structure.
3617 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3618 * out-of-sync. Make sure to update the required fields
3619 * before using them.
3620 *
3621 * @remarks No-long-jump zone!!!
3622 */
3623DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3624{
3625 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3626 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3627 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3628 AssertRCReturn(rc, rc);
3629 return rc;
3630}
3631
3632
3633/**
3634 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3635 * CR0 is partially shared with the host and we have to consider the FPU bits.
3636 *
3637 * @returns VBox status code.
3638 * @param pVCpu The cross context virtual CPU structure.
3639 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3640 * out-of-sync. Make sure to update the required fields
3641 * before using them.
3642 *
3643 * @remarks No-long-jump zone!!!
3644 */
3645static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3646{
3647 /*
3648 * Guest CR0.
3649 * Guest FPU.
3650 */
3651 int rc = VINF_SUCCESS;
3652 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3653 {
3654 Assert(!(pMixedCtx->cr0 >> 32));
3655 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3656 PVM pVM = pVCpu->CTX_SUFF(pVM);
3657
3658 /* The guest's view (read access) of its CR0 is unblemished. */
3659 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3660 AssertRCReturn(rc, rc);
3661 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3662
3663 /* Setup VT-x's view of the guest CR0. */
3664 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3665 if (pVM->hm.s.fNestedPaging)
3666 {
3667 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3668 {
3669 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3670 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3671 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3672 }
3673 else
3674 {
3675 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3676 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3677 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3678 }
3679
3680 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3681 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3682 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3683
3684 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3685 AssertRCReturn(rc, rc);
3686 }
3687 else
3688 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3689
3690 /*
3691 * Guest FPU bits.
3692 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3693 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3694 */
3695 u32GuestCR0 |= X86_CR0_NE;
3696 bool fInterceptNM = false;
3697 if (CPUMIsGuestFPUStateActive(pVCpu))
3698 {
3699 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3700 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3701 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3702 }
3703 else
3704 {
3705 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3706 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3707 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3708 }
3709
3710 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3711 bool fInterceptMF = false;
3712 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3713 fInterceptMF = true;
3714
3715 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3716 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3717 {
3718 Assert(PDMVmmDevHeapIsEnabled(pVM));
3719 Assert(pVM->hm.s.vmx.pRealModeTSS);
3720 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3721 fInterceptNM = true;
3722 fInterceptMF = true;
3723 }
3724 else
3725 {
3726 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3727 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3728 }
3729 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3730
3731 if (fInterceptNM)
3732 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3733 else
3734 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3735
3736 if (fInterceptMF)
3737 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3738 else
3739 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3740
3741 /* Additional intercepts for debugging, define these yourself explicitly. */
3742#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3743 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3744 | RT_BIT(X86_XCPT_BP)
3745 | RT_BIT(X86_XCPT_DE)
3746 | RT_BIT(X86_XCPT_NM)
3747 | RT_BIT(X86_XCPT_TS)
3748 | RT_BIT(X86_XCPT_UD)
3749 | RT_BIT(X86_XCPT_NP)
3750 | RT_BIT(X86_XCPT_SS)
3751 | RT_BIT(X86_XCPT_GP)
3752 | RT_BIT(X86_XCPT_PF)
3753 | RT_BIT(X86_XCPT_MF)
3754 ;
3755#elif defined(HMVMX_ALWAYS_TRAP_PF)
3756 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3757#endif
3758
3759 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3760
3761 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3762 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3763 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3764 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3765 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3766 else
3767 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3768
3769 u32GuestCR0 |= uSetCR0;
3770 u32GuestCR0 &= uZapCR0;
3771 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3772
3773 /* Write VT-x's view of the guest CR0 into the VMCS. */
3774 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3775 AssertRCReturn(rc, rc);
3776 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3777 uZapCR0));
3778
3779 /*
3780 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3781 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3782 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3783 */
3784 uint32_t u32CR0Mask = 0;
3785 u32CR0Mask = X86_CR0_PE
3786 | X86_CR0_NE
3787 | X86_CR0_WP
3788 | X86_CR0_PG
3789 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3790 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3791 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3792
3793 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3794 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3795 * and @bugref{6944}. */
3796#if 0
3797 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3798 u32CR0Mask &= ~X86_CR0_PE;
3799#endif
3800 if (pVM->hm.s.fNestedPaging)
3801 u32CR0Mask &= ~X86_CR0_WP;
3802
3803 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3804 if (fInterceptNM)
3805 {
3806 u32CR0Mask |= X86_CR0_TS
3807 | X86_CR0_MP;
3808 }
3809
3810 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3811 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3812 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3813 AssertRCReturn(rc, rc);
3814 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3815
3816 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3817 }
3818 return rc;
3819}
3820
3821
3822/**
3823 * Loads the guest control registers (CR3, CR4) into the guest-state area
3824 * in the VMCS.
3825 *
3826 * @returns VBox status code.
3827 * @param pVCpu The cross context virtual CPU structure.
3828 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3829 * out-of-sync. Make sure to update the required fields
3830 * before using them.
3831 *
3832 * @remarks No-long-jump zone!!!
3833 */
3834static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3835{
3836 int rc = VINF_SUCCESS;
3837 PVM pVM = pVCpu->CTX_SUFF(pVM);
3838
3839 /*
3840 * Guest CR2.
3841 * It's always loaded in the assembler code. Nothing to do here.
3842 */
3843
3844 /*
3845 * Guest CR3.
3846 */
3847 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3848 {
3849 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3850 if (pVM->hm.s.fNestedPaging)
3851 {
3852 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3853
3854 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3855 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3856 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3857 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3858
3859 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3860 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3861 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3862
3863 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3864 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3865 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3866 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3867 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3868 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3869 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3870
3871 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3872 AssertRCReturn(rc, rc);
3873 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3874
3875 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3876 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3877 {
3878 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3879 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3880 {
3881 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3882 AssertRCReturn(rc, rc);
3883 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3884 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3885 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3886 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3887 AssertRCReturn(rc, rc);
3888 }
3889
3890 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3891 have Unrestricted Execution to handle the guest when it's not using paging. */
3892 GCPhysGuestCR3 = pMixedCtx->cr3;
3893 }
3894 else
3895 {
3896 /*
3897 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3898 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3899 * EPT takes care of translating it to host-physical addresses.
3900 */
3901 RTGCPHYS GCPhys;
3902 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3903 Assert(PDMVmmDevHeapIsEnabled(pVM));
3904
3905 /* We obtain it here every time as the guest could have relocated this PCI region. */
3906 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3907 AssertRCReturn(rc, rc);
3908
3909 GCPhysGuestCR3 = GCPhys;
3910 }
3911
3912 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3913 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3914 }
3915 else
3916 {
3917 /* Non-nested paging case, just use the hypervisor's CR3. */
3918 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3919
3920 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3921 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3922 }
3923 AssertRCReturn(rc, rc);
3924
3925 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3926 }
3927
3928 /*
3929 * Guest CR4.
3930 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3931 */
3932 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3933 {
3934 Assert(!(pMixedCtx->cr4 >> 32));
3935 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3936
3937 /* The guest's view of its CR4 is unblemished. */
3938 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3939 AssertRCReturn(rc, rc);
3940 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3941
3942 /* Setup VT-x's view of the guest CR4. */
3943 /*
3944 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3945 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3946 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3947 */
3948 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3949 {
3950 Assert(pVM->hm.s.vmx.pRealModeTSS);
3951 Assert(PDMVmmDevHeapIsEnabled(pVM));
3952 u32GuestCR4 &= ~X86_CR4_VME;
3953 }
3954
3955 if (pVM->hm.s.fNestedPaging)
3956 {
3957 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3958 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3959 {
3960 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3961 u32GuestCR4 |= X86_CR4_PSE;
3962 /* Our identity mapping is a 32-bit page directory. */
3963 u32GuestCR4 &= ~X86_CR4_PAE;
3964 }
3965 /* else use guest CR4.*/
3966 }
3967 else
3968 {
3969 /*
3970 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3971 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3972 */
3973 switch (pVCpu->hm.s.enmShadowMode)
3974 {
3975 case PGMMODE_REAL: /* Real-mode. */
3976 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3977 case PGMMODE_32_BIT: /* 32-bit paging. */
3978 {
3979 u32GuestCR4 &= ~X86_CR4_PAE;
3980 break;
3981 }
3982
3983 case PGMMODE_PAE: /* PAE paging. */
3984 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3985 {
3986 u32GuestCR4 |= X86_CR4_PAE;
3987 break;
3988 }
3989
3990 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3991 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3992#ifdef VBOX_ENABLE_64_BITS_GUESTS
3993 break;
3994#endif
3995 default:
3996 AssertFailed();
3997 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3998 }
3999 }
4000
4001 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4002 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4003 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4004 u32GuestCR4 |= uSetCR4;
4005 u32GuestCR4 &= uZapCR4;
4006
4007 /* Write VT-x's view of the guest CR4 into the VMCS. */
4008 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4009 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4010 AssertRCReturn(rc, rc);
4011
4012 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4013 uint32_t u32CR4Mask = X86_CR4_VME
4014 | X86_CR4_PAE
4015 | X86_CR4_PGE
4016 | X86_CR4_PSE
4017 | X86_CR4_VMXE;
4018 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4019 u32CR4Mask |= X86_CR4_OSXSAVE;
4020 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4021 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4022 AssertRCReturn(rc, rc);
4023
4024 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4025 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4026
4027 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4028 }
4029 return rc;
4030}
4031
4032
4033/**
4034 * Loads the guest debug registers into the guest-state area in the VMCS.
4035 *
4036 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4037 *
4038 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4039 *
4040 * @returns VBox status code.
4041 * @param pVCpu The cross context virtual CPU structure.
4042 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4043 * out-of-sync. Make sure to update the required fields
4044 * before using them.
4045 *
4046 * @remarks No-long-jump zone!!!
4047 */
4048static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4049{
4050 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4051 return VINF_SUCCESS;
4052
4053#ifdef VBOX_STRICT
4054 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4055 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4056 {
4057 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4058 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4059 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4060 }
4061#endif
4062
4063 int rc;
4064 PVM pVM = pVCpu->CTX_SUFF(pVM);
4065 bool fSteppingDB = false;
4066 bool fInterceptMovDRx = false;
4067 if (pVCpu->hm.s.fSingleInstruction)
4068 {
4069 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4070 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4071 {
4072 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4073 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4074 AssertRCReturn(rc, rc);
4075 Assert(fSteppingDB == false);
4076 }
4077 else
4078 {
4079 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4080 pVCpu->hm.s.fClearTrapFlag = true;
4081 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4082 fSteppingDB = true;
4083 }
4084 }
4085
4086 if ( fSteppingDB
4087 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4088 {
4089 /*
4090 * Use the combined guest and host DRx values found in the hypervisor
4091 * register set because the debugger has breakpoints active or someone
4092 * is single stepping on the host side without a monitor trap flag.
4093 *
4094 * Note! DBGF expects a clean DR6 state before executing guest code.
4095 */
4096#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4097 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4098 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4099 {
4100 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4101 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4102 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4103 }
4104 else
4105#endif
4106 if (!CPUMIsHyperDebugStateActive(pVCpu))
4107 {
4108 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4109 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4110 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4111 }
4112
4113 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4114 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4115 AssertRCReturn(rc, rc);
4116
4117 pVCpu->hm.s.fUsingHyperDR7 = true;
4118 fInterceptMovDRx = true;
4119 }
4120 else
4121 {
4122 /*
4123 * If the guest has enabled debug registers, we need to load them prior to
4124 * executing guest code so they'll trigger at the right time.
4125 */
4126 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4127 {
4128#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4129 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4130 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4131 {
4132 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4133 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4134 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4135 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4136 }
4137 else
4138#endif
4139 if (!CPUMIsGuestDebugStateActive(pVCpu))
4140 {
4141 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4142 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4143 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4144 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4145 }
4146 Assert(!fInterceptMovDRx);
4147 }
4148 /*
4149 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4150 * must intercept #DB in order to maintain a correct DR6 guest value, and
4151 * because we need to intercept it to prevent nested #DBs from hanging the
4152 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4153 */
4154#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4155 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4156 && !CPUMIsGuestDebugStateActive(pVCpu))
4157#else
4158 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4159#endif
4160 {
4161 fInterceptMovDRx = true;
4162 }
4163
4164 /* Update guest DR7. */
4165 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4166 AssertRCReturn(rc, rc);
4167
4168 pVCpu->hm.s.fUsingHyperDR7 = false;
4169 }
4170
4171 /*
4172 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4173 */
4174 if (fInterceptMovDRx)
4175 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4176 else
4177 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4178 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4179 AssertRCReturn(rc, rc);
4180
4181 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4182 return VINF_SUCCESS;
4183}
4184
4185
4186#ifdef VBOX_STRICT
4187/**
4188 * Strict function to validate segment registers.
4189 *
4190 * @remarks ASSUMES CR0 is up to date.
4191 */
4192static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4193{
4194 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4195 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4196 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4197 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4198 && ( !CPUMIsGuestInRealModeEx(pCtx)
4199 && !CPUMIsGuestInV86ModeEx(pCtx)))
4200 {
4201 /* Protected mode checks */
4202 /* CS */
4203 Assert(pCtx->cs.Attr.n.u1Present);
4204 Assert(!(pCtx->cs.Attr.u & 0xf00));
4205 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4206 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4207 || !(pCtx->cs.Attr.n.u1Granularity));
4208 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4209 || (pCtx->cs.Attr.n.u1Granularity));
4210 /* CS cannot be loaded with NULL in protected mode. */
4211 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4212 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4213 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4214 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4215 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4216 else
4217 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4218 /* SS */
4219 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4220 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4221 if ( !(pCtx->cr0 & X86_CR0_PE)
4222 || pCtx->cs.Attr.n.u4Type == 3)
4223 {
4224 Assert(!pCtx->ss.Attr.n.u2Dpl);
4225 }
4226 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4227 {
4228 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4229 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4230 Assert(pCtx->ss.Attr.n.u1Present);
4231 Assert(!(pCtx->ss.Attr.u & 0xf00));
4232 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4233 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4234 || !(pCtx->ss.Attr.n.u1Granularity));
4235 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4236 || (pCtx->ss.Attr.n.u1Granularity));
4237 }
4238 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4239 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4240 {
4241 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4242 Assert(pCtx->ds.Attr.n.u1Present);
4243 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4244 Assert(!(pCtx->ds.Attr.u & 0xf00));
4245 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4246 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4247 || !(pCtx->ds.Attr.n.u1Granularity));
4248 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4249 || (pCtx->ds.Attr.n.u1Granularity));
4250 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4251 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4252 }
4253 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4254 {
4255 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4256 Assert(pCtx->es.Attr.n.u1Present);
4257 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4258 Assert(!(pCtx->es.Attr.u & 0xf00));
4259 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4260 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4261 || !(pCtx->es.Attr.n.u1Granularity));
4262 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4263 || (pCtx->es.Attr.n.u1Granularity));
4264 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4265 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4266 }
4267 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4268 {
4269 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4270 Assert(pCtx->fs.Attr.n.u1Present);
4271 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4272 Assert(!(pCtx->fs.Attr.u & 0xf00));
4273 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4274 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4275 || !(pCtx->fs.Attr.n.u1Granularity));
4276 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4277 || (pCtx->fs.Attr.n.u1Granularity));
4278 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4279 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4280 }
4281 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4282 {
4283 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4284 Assert(pCtx->gs.Attr.n.u1Present);
4285 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4286 Assert(!(pCtx->gs.Attr.u & 0xf00));
4287 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4288 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4289 || !(pCtx->gs.Attr.n.u1Granularity));
4290 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4291 || (pCtx->gs.Attr.n.u1Granularity));
4292 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4293 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4294 }
4295 /* 64-bit capable CPUs. */
4296# if HC_ARCH_BITS == 64
4297 Assert(!(pCtx->cs.u64Base >> 32));
4298 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4299 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4300 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4301# endif
4302 }
4303 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4304 || ( CPUMIsGuestInRealModeEx(pCtx)
4305 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4306 {
4307 /* Real and v86 mode checks. */
4308 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4309 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4310 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4311 {
4312 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4313 }
4314 else
4315 {
4316 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4317 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4318 }
4319
4320 /* CS */
4321 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4322 Assert(pCtx->cs.u32Limit == 0xffff);
4323 Assert(u32CSAttr == 0xf3);
4324 /* SS */
4325 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4326 Assert(pCtx->ss.u32Limit == 0xffff);
4327 Assert(u32SSAttr == 0xf3);
4328 /* DS */
4329 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4330 Assert(pCtx->ds.u32Limit == 0xffff);
4331 Assert(u32DSAttr == 0xf3);
4332 /* ES */
4333 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4334 Assert(pCtx->es.u32Limit == 0xffff);
4335 Assert(u32ESAttr == 0xf3);
4336 /* FS */
4337 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4338 Assert(pCtx->fs.u32Limit == 0xffff);
4339 Assert(u32FSAttr == 0xf3);
4340 /* GS */
4341 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4342 Assert(pCtx->gs.u32Limit == 0xffff);
4343 Assert(u32GSAttr == 0xf3);
4344 /* 64-bit capable CPUs. */
4345# if HC_ARCH_BITS == 64
4346 Assert(!(pCtx->cs.u64Base >> 32));
4347 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4348 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4349 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4350# endif
4351 }
4352}
4353#endif /* VBOX_STRICT */
4354
4355
4356/**
4357 * Writes a guest segment register into the guest-state area in the VMCS.
4358 *
4359 * @returns VBox status code.
4360 * @param pVCpu The cross context virtual CPU structure.
4361 * @param idxSel Index of the selector in the VMCS.
4362 * @param idxLimit Index of the segment limit in the VMCS.
4363 * @param idxBase Index of the segment base in the VMCS.
4364 * @param idxAccess Index of the access rights of the segment in the VMCS.
4365 * @param pSelReg Pointer to the segment selector.
4366 *
4367 * @remarks No-long-jump zone!!!
4368 */
4369static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4370 uint32_t idxAccess, PCPUMSELREG pSelReg)
4371{
4372 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4373 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4374 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4375 AssertRCReturn(rc, rc);
4376
4377 uint32_t u32Access = pSelReg->Attr.u;
4378 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4379 {
4380 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4381 u32Access = 0xf3;
4382 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4383 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4384 }
4385 else
4386 {
4387 /*
4388 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4389 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4390 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4391 * loaded in protected-mode have their attribute as 0.
4392 */
4393 if (!u32Access)
4394 u32Access = X86DESCATTR_UNUSABLE;
4395 }
4396
4397 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4398 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4399 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4400
4401 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4402 AssertRCReturn(rc, rc);
4403 return rc;
4404}
4405
4406
4407/**
4408 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4409 * into the guest-state area in the VMCS.
4410 *
4411 * @returns VBox status code.
4412 * @param pVCpu The cross context virtual CPU structure.
4413 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4414 * out-of-sync. Make sure to update the required fields
4415 * before using them.
4416 *
4417 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4418 * @remarks No-long-jump zone!!!
4419 */
4420static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4421{
4422 int rc = VERR_INTERNAL_ERROR_5;
4423 PVM pVM = pVCpu->CTX_SUFF(pVM);
4424
4425 /*
4426 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4427 */
4428 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4429 {
4430 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4431 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4432 {
4433 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4434 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4435 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4436 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4437 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4438 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4439 }
4440
4441#ifdef VBOX_WITH_REM
4442 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4443 {
4444 Assert(pVM->hm.s.vmx.pRealModeTSS);
4445 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4446 if ( pVCpu->hm.s.vmx.fWasInRealMode
4447 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4448 {
4449 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4450 in real-mode (e.g. OpenBSD 4.0) */
4451 REMFlushTBs(pVM);
4452 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4453 pVCpu->hm.s.vmx.fWasInRealMode = false;
4454 }
4455 }
4456#endif
4457 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4458 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4459 AssertRCReturn(rc, rc);
4460 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4461 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4462 AssertRCReturn(rc, rc);
4463 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4464 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4465 AssertRCReturn(rc, rc);
4466 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4467 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4468 AssertRCReturn(rc, rc);
4469 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4470 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4471 AssertRCReturn(rc, rc);
4472 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4473 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4474 AssertRCReturn(rc, rc);
4475
4476#ifdef VBOX_STRICT
4477 /* Validate. */
4478 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4479#endif
4480
4481 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4482 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4483 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4484 }
4485
4486 /*
4487 * Guest TR.
4488 */
4489 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4490 {
4491 /*
4492 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4493 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4494 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4495 */
4496 uint16_t u16Sel = 0;
4497 uint32_t u32Limit = 0;
4498 uint64_t u64Base = 0;
4499 uint32_t u32AccessRights = 0;
4500
4501 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4502 {
4503 u16Sel = pMixedCtx->tr.Sel;
4504 u32Limit = pMixedCtx->tr.u32Limit;
4505 u64Base = pMixedCtx->tr.u64Base;
4506 u32AccessRights = pMixedCtx->tr.Attr.u;
4507 }
4508 else
4509 {
4510 Assert(pVM->hm.s.vmx.pRealModeTSS);
4511 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4512
4513 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4514 RTGCPHYS GCPhys;
4515 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4516 AssertRCReturn(rc, rc);
4517
4518 X86DESCATTR DescAttr;
4519 DescAttr.u = 0;
4520 DescAttr.n.u1Present = 1;
4521 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4522
4523 u16Sel = 0;
4524 u32Limit = HM_VTX_TSS_SIZE;
4525 u64Base = GCPhys; /* in real-mode phys = virt. */
4526 u32AccessRights = DescAttr.u;
4527 }
4528
4529 /* Validate. */
4530 Assert(!(u16Sel & RT_BIT(2)));
4531 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4532 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4533 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4534 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4535 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4536 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4537 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4538 Assert( (u32Limit & 0xfff) == 0xfff
4539 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4540 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4541 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4542
4543 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4544 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4545 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4546 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4547 AssertRCReturn(rc, rc);
4548
4549 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4550 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4551 }
4552
4553 /*
4554 * Guest GDTR.
4555 */
4556 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4557 {
4558 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4559 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4560 AssertRCReturn(rc, rc);
4561
4562 /* Validate. */
4563 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4564
4565 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4566 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4567 }
4568
4569 /*
4570 * Guest LDTR.
4571 */
4572 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4573 {
4574 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4575 uint32_t u32Access = 0;
4576 if (!pMixedCtx->ldtr.Attr.u)
4577 u32Access = X86DESCATTR_UNUSABLE;
4578 else
4579 u32Access = pMixedCtx->ldtr.Attr.u;
4580
4581 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4582 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4583 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4584 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4585 AssertRCReturn(rc, rc);
4586
4587 /* Validate. */
4588 if (!(u32Access & X86DESCATTR_UNUSABLE))
4589 {
4590 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4591 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4592 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4593 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4594 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4595 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4596 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4597 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4598 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4599 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4600 }
4601
4602 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4603 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4604 }
4605
4606 /*
4607 * Guest IDTR.
4608 */
4609 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4610 {
4611 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4612 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4613 AssertRCReturn(rc, rc);
4614
4615 /* Validate. */
4616 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4617
4618 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4619 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4620 }
4621
4622 return VINF_SUCCESS;
4623}
4624
4625
4626/**
4627 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4628 * areas.
4629 *
4630 * These MSRs will automatically be loaded to the host CPU on every successful
4631 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4632 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4633 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4634 *
4635 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4636 *
4637 * @returns VBox status code.
4638 * @param pVCpu The cross context virtual CPU structure.
4639 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4640 * out-of-sync. Make sure to update the required fields
4641 * before using them.
4642 *
4643 * @remarks No-long-jump zone!!!
4644 */
4645static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4646{
4647 AssertPtr(pVCpu);
4648 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4649
4650 /*
4651 * MSRs that we use the auto-load/store MSR area in the VMCS.
4652 */
4653 PVM pVM = pVCpu->CTX_SUFF(pVM);
4654 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4655 {
4656 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4657#if HC_ARCH_BITS == 32
4658 if (pVM->hm.s.fAllow64BitGuests)
4659 {
4660 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4661 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4662 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4663 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4664 AssertRCReturn(rc, rc);
4665# ifdef LOG_ENABLED
4666 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4667 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4668 {
4669 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4670 pMsr->u64Value));
4671 }
4672# endif
4673 }
4674#endif
4675 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4676 }
4677
4678 /*
4679 * Guest Sysenter MSRs.
4680 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4681 * VM-exits on WRMSRs for these MSRs.
4682 */
4683 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4684 {
4685 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4686 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4687 }
4688
4689 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4690 {
4691 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4692 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4693 }
4694
4695 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4696 {
4697 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4698 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4699 }
4700
4701 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4702 {
4703 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4704 {
4705 /*
4706 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4707 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4708 */
4709 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4710 {
4711 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4712 AssertRCReturn(rc,rc);
4713 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4714 }
4715 else
4716 {
4717 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4718 NULL /* pfAddedAndUpdated */);
4719 AssertRCReturn(rc, rc);
4720
4721 /* We need to intercept reads too, see @bugref{7386#c16}. */
4722 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4723 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4724 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4725 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4726 }
4727 }
4728 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4729 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4730 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4731 }
4732
4733 return VINF_SUCCESS;
4734}
4735
4736
4737/**
4738 * Loads the guest activity state into the guest-state area in the VMCS.
4739 *
4740 * @returns VBox status code.
4741 * @param pVCpu The cross context virtual CPU structure.
4742 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4743 * out-of-sync. Make sure to update the required fields
4744 * before using them.
4745 *
4746 * @remarks No-long-jump zone!!!
4747 */
4748static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4749{
4750 NOREF(pMixedCtx);
4751 /** @todo See if we can make use of other states, e.g.
4752 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4753 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4754 {
4755 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4756 AssertRCReturn(rc, rc);
4757
4758 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4759 }
4760 return VINF_SUCCESS;
4761}
4762
4763
4764/**
4765 * Sets up the appropriate function to run guest code.
4766 *
4767 * @returns VBox status code.
4768 * @param pVCpu The cross context virtual CPU structure.
4769 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4770 * out-of-sync. Make sure to update the required fields
4771 * before using them.
4772 *
4773 * @remarks No-long-jump zone!!!
4774 */
4775static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4776{
4777 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4778 {
4779#ifndef VBOX_ENABLE_64_BITS_GUESTS
4780 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4781#endif
4782 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4783#if HC_ARCH_BITS == 32
4784 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4785 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4786 {
4787 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4788 {
4789 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4790 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4791 | HM_CHANGED_VMX_ENTRY_CTLS
4792 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4793 }
4794 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4795 }
4796#else
4797 /* 64-bit host. */
4798 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4799#endif
4800 }
4801 else
4802 {
4803 /* Guest is not in long mode, use the 32-bit handler. */
4804#if HC_ARCH_BITS == 32
4805 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4806 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4807 {
4808 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4809 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4810 | HM_CHANGED_VMX_ENTRY_CTLS
4811 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4812 }
4813#endif
4814 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4815 }
4816 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4817 return VINF_SUCCESS;
4818}
4819
4820
4821/**
4822 * Wrapper for running the guest code in VT-x.
4823 *
4824 * @returns VBox status code, no informational status codes.
4825 * @param pVM The cross context VM structure.
4826 * @param pVCpu The cross context virtual CPU structure.
4827 * @param pCtx Pointer to the guest-CPU context.
4828 *
4829 * @remarks No-long-jump zone!!!
4830 */
4831DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4832{
4833 /*
4834 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4835 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4836 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4837 */
4838 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4839 /** @todo Add stats for resume vs launch. */
4840#ifdef VBOX_WITH_KERNEL_USING_XMM
4841 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4842#else
4843 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4844#endif
4845 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4846 return rc;
4847}
4848
4849
4850/**
4851 * Reports world-switch error and dumps some useful debug info.
4852 *
4853 * @param pVM The cross context VM structure.
4854 * @param pVCpu The cross context virtual CPU structure.
4855 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4856 * @param pCtx Pointer to the guest-CPU context.
4857 * @param pVmxTransient Pointer to the VMX transient structure (only
4858 * exitReason updated).
4859 */
4860static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4861{
4862 Assert(pVM);
4863 Assert(pVCpu);
4864 Assert(pCtx);
4865 Assert(pVmxTransient);
4866 HMVMX_ASSERT_PREEMPT_SAFE();
4867
4868 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4869 switch (rcVMRun)
4870 {
4871 case VERR_VMX_INVALID_VMXON_PTR:
4872 AssertFailed();
4873 break;
4874 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4875 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4876 {
4877 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4878 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4879 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4880 AssertRC(rc);
4881
4882 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4883 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4884 Cannot do it here as we may have been long preempted. */
4885
4886#ifdef VBOX_STRICT
4887 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4888 pVmxTransient->uExitReason));
4889 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4890 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4891 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4892 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4893 else
4894 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4895 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4896 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4897
4898 /* VMX control bits. */
4899 uint32_t u32Val;
4900 uint64_t u64Val;
4901 RTHCUINTREG uHCReg;
4902 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4903 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4904 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4905 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4906 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4907 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4908 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4909 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4910 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4911 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4912 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4913 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4914 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4915 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4916 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4917 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4918 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4919 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4920 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4921 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4922 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4923 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4924 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4925 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4926 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4927 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4928 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4929 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4930 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4931 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4932 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4933 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4934 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4935 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4936 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4937 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4938 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4939 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4940 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4941 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4942 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4943 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4944
4945 /* Guest bits. */
4946 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4947 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4948 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4949 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4950 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4951 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4952 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
4953 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
4954
4955 /* Host bits. */
4956 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4957 Log4(("Host CR0 %#RHr\n", uHCReg));
4958 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4959 Log4(("Host CR3 %#RHr\n", uHCReg));
4960 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4961 Log4(("Host CR4 %#RHr\n", uHCReg));
4962
4963 RTGDTR HostGdtr;
4964 PCX86DESCHC pDesc;
4965 ASMGetGDTR(&HostGdtr);
4966 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
4967 Log4(("Host CS %#08x\n", u32Val));
4968 if (u32Val < HostGdtr.cbGdt)
4969 {
4970 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4971 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4972 }
4973
4974 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
4975 Log4(("Host DS %#08x\n", u32Val));
4976 if (u32Val < HostGdtr.cbGdt)
4977 {
4978 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4979 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4980 }
4981
4982 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
4983 Log4(("Host ES %#08x\n", u32Val));
4984 if (u32Val < HostGdtr.cbGdt)
4985 {
4986 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4987 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4988 }
4989
4990 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
4991 Log4(("Host FS %#08x\n", u32Val));
4992 if (u32Val < HostGdtr.cbGdt)
4993 {
4994 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4995 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4996 }
4997
4998 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
4999 Log4(("Host GS %#08x\n", u32Val));
5000 if (u32Val < HostGdtr.cbGdt)
5001 {
5002 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5003 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5004 }
5005
5006 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5007 Log4(("Host SS %#08x\n", u32Val));
5008 if (u32Val < HostGdtr.cbGdt)
5009 {
5010 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5011 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5012 }
5013
5014 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5015 Log4(("Host TR %#08x\n", u32Val));
5016 if (u32Val < HostGdtr.cbGdt)
5017 {
5018 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5019 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5020 }
5021
5022 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5023 Log4(("Host TR Base %#RHv\n", uHCReg));
5024 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5025 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5026 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5027 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5028 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5029 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5030 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5031 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5032 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5033 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5034 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5035 Log4(("Host RSP %#RHv\n", uHCReg));
5036 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5037 Log4(("Host RIP %#RHv\n", uHCReg));
5038# if HC_ARCH_BITS == 64
5039 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5040 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5041 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5042 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5043 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5044 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5045# endif
5046#endif /* VBOX_STRICT */
5047 break;
5048 }
5049
5050 default:
5051 /* Impossible */
5052 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5053 break;
5054 }
5055 NOREF(pVM); NOREF(pCtx);
5056}
5057
5058
5059#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5060#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5061# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5062#endif
5063#ifdef VBOX_STRICT
5064static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5065{
5066 switch (idxField)
5067 {
5068 case VMX_VMCS_GUEST_RIP:
5069 case VMX_VMCS_GUEST_RSP:
5070 case VMX_VMCS_GUEST_SYSENTER_EIP:
5071 case VMX_VMCS_GUEST_SYSENTER_ESP:
5072 case VMX_VMCS_GUEST_GDTR_BASE:
5073 case VMX_VMCS_GUEST_IDTR_BASE:
5074 case VMX_VMCS_GUEST_CS_BASE:
5075 case VMX_VMCS_GUEST_DS_BASE:
5076 case VMX_VMCS_GUEST_ES_BASE:
5077 case VMX_VMCS_GUEST_FS_BASE:
5078 case VMX_VMCS_GUEST_GS_BASE:
5079 case VMX_VMCS_GUEST_SS_BASE:
5080 case VMX_VMCS_GUEST_LDTR_BASE:
5081 case VMX_VMCS_GUEST_TR_BASE:
5082 case VMX_VMCS_GUEST_CR3:
5083 return true;
5084 }
5085 return false;
5086}
5087
5088static bool hmR0VmxIsValidReadField(uint32_t idxField)
5089{
5090 switch (idxField)
5091 {
5092 /* Read-only fields. */
5093 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5094 return true;
5095 }
5096 /* Remaining readable fields should also be writable. */
5097 return hmR0VmxIsValidWriteField(idxField);
5098}
5099#endif /* VBOX_STRICT */
5100
5101
5102/**
5103 * Executes the specified handler in 64-bit mode.
5104 *
5105 * @returns VBox status code (no informational status codes).
5106 * @param pVM The cross context VM structure.
5107 * @param pVCpu The cross context virtual CPU structure.
5108 * @param pCtx Pointer to the guest CPU context.
5109 * @param enmOp The operation to perform.
5110 * @param cParams Number of parameters.
5111 * @param paParam Array of 32-bit parameters.
5112 */
5113VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5114 uint32_t cParams, uint32_t *paParam)
5115{
5116 NOREF(pCtx);
5117
5118 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5119 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5120 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5121 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5122
5123#ifdef VBOX_STRICT
5124 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5125 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5126
5127 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5128 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5129#endif
5130
5131 /* Disable interrupts. */
5132 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5133
5134#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5135 RTCPUID idHostCpu = RTMpCpuId();
5136 CPUMR0SetLApic(pVCpu, idHostCpu);
5137#endif
5138
5139 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5140 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5141
5142 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5143 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5144
5145 /* Leave VMX Root Mode. */
5146 VMXDisable();
5147
5148 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5149
5150 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5151 CPUMSetHyperEIP(pVCpu, enmOp);
5152 for (int i = (int)cParams - 1; i >= 0; i--)
5153 CPUMPushHyper(pVCpu, paParam[i]);
5154
5155 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5156
5157 /* Call the switcher. */
5158 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5159 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5160
5161 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5162 /* Make sure the VMX instructions don't cause #UD faults. */
5163 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5164
5165 /* Re-enter VMX Root Mode */
5166 int rc2 = VMXEnable(HCPhysCpuPage);
5167 if (RT_FAILURE(rc2))
5168 {
5169 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5170 ASMSetFlags(fOldEFlags);
5171 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5172 return rc2;
5173 }
5174
5175 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5176 AssertRC(rc2);
5177 Assert(!(ASMGetFlags() & X86_EFL_IF));
5178 ASMSetFlags(fOldEFlags);
5179 return rc;
5180}
5181
5182
5183/**
5184 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5185 * supporting 64-bit guests.
5186 *
5187 * @returns VBox status code.
5188 * @param fResume Whether to VMLAUNCH or VMRESUME.
5189 * @param pCtx Pointer to the guest-CPU context.
5190 * @param pCache Pointer to the VMCS cache.
5191 * @param pVM The cross context VM structure.
5192 * @param pVCpu The cross context virtual CPU structure.
5193 */
5194DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5195{
5196 NOREF(fResume);
5197
5198 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5199 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5200
5201#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5202 pCache->uPos = 1;
5203 pCache->interPD = PGMGetInterPaeCR3(pVM);
5204 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5205#endif
5206
5207#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5208 pCache->TestIn.HCPhysCpuPage = 0;
5209 pCache->TestIn.HCPhysVmcs = 0;
5210 pCache->TestIn.pCache = 0;
5211 pCache->TestOut.HCPhysVmcs = 0;
5212 pCache->TestOut.pCache = 0;
5213 pCache->TestOut.pCtx = 0;
5214 pCache->TestOut.eflags = 0;
5215#else
5216 NOREF(pCache);
5217#endif
5218
5219 uint32_t aParam[10];
5220 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5221 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5222 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5223 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5224 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5225 aParam[5] = 0;
5226 aParam[6] = VM_RC_ADDR(pVM, pVM);
5227 aParam[7] = 0;
5228 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5229 aParam[9] = 0;
5230
5231#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5232 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5233 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5234#endif
5235 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5236
5237#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5238 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5239 Assert(pCtx->dr[4] == 10);
5240 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5241#endif
5242
5243#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5244 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5245 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5246 pVCpu->hm.s.vmx.HCPhysVmcs));
5247 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5248 pCache->TestOut.HCPhysVmcs));
5249 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5250 pCache->TestOut.pCache));
5251 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5252 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5253 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5254 pCache->TestOut.pCtx));
5255 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5256#endif
5257 return rc;
5258}
5259
5260
5261/**
5262 * Initialize the VMCS-Read cache.
5263 *
5264 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5265 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5266 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5267 * (those that have a 32-bit FULL & HIGH part).
5268 *
5269 * @returns VBox status code.
5270 * @param pVM The cross context VM structure.
5271 * @param pVCpu The cross context virtual CPU structure.
5272 */
5273static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5274{
5275#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5276{ \
5277 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5278 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5279 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5280 ++cReadFields; \
5281}
5282
5283 AssertPtr(pVM);
5284 AssertPtr(pVCpu);
5285 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5286 uint32_t cReadFields = 0;
5287
5288 /*
5289 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5290 * and serve to indicate exceptions to the rules.
5291 */
5292
5293 /* Guest-natural selector base fields. */
5294#if 0
5295 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5296 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5297 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5298#endif
5299 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5300 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5301 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5302 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5303 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5304 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5305 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5306 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5307 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5308 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5309 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5310 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5311#if 0
5312 /* Unused natural width guest-state fields. */
5313 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5314 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5315#endif
5316 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5317 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5318
5319 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5320#if 0
5321 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5322 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5323 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5324 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5325 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5326 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5327 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5328 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5329 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5330#endif
5331
5332 /* Natural width guest-state fields. */
5333 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5334#if 0
5335 /* Currently unused field. */
5336 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5337#endif
5338
5339 if (pVM->hm.s.fNestedPaging)
5340 {
5341 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5342 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5343 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5344 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5345 }
5346 else
5347 {
5348 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5349 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5350 }
5351
5352#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5353 return VINF_SUCCESS;
5354}
5355
5356
5357/**
5358 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5359 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5360 * darwin, running 64-bit guests).
5361 *
5362 * @returns VBox status code.
5363 * @param pVCpu The cross context virtual CPU structure.
5364 * @param idxField The VMCS field encoding.
5365 * @param u64Val 16, 32 or 64-bit value.
5366 */
5367VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5368{
5369 int rc;
5370 switch (idxField)
5371 {
5372 /*
5373 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5374 */
5375 /* 64-bit Control fields. */
5376 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5377 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5378 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5379 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5380 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5381 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5382 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5383 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5384 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5385 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5386 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5387 case VMX_VMCS64_CTRL_EPTP_FULL:
5388 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5389 /* 64-bit Guest-state fields. */
5390 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5391 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5392 case VMX_VMCS64_GUEST_PAT_FULL:
5393 case VMX_VMCS64_GUEST_EFER_FULL:
5394 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5395 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5396 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5397 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5398 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5399 /* 64-bit Host-state fields. */
5400 case VMX_VMCS64_HOST_PAT_FULL:
5401 case VMX_VMCS64_HOST_EFER_FULL:
5402 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5403 {
5404 rc = VMXWriteVmcs32(idxField, u64Val);
5405 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5406 break;
5407 }
5408
5409 /*
5410 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5411 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5412 */
5413 /* Natural-width Guest-state fields. */
5414 case VMX_VMCS_GUEST_CR3:
5415 case VMX_VMCS_GUEST_ES_BASE:
5416 case VMX_VMCS_GUEST_CS_BASE:
5417 case VMX_VMCS_GUEST_SS_BASE:
5418 case VMX_VMCS_GUEST_DS_BASE:
5419 case VMX_VMCS_GUEST_FS_BASE:
5420 case VMX_VMCS_GUEST_GS_BASE:
5421 case VMX_VMCS_GUEST_LDTR_BASE:
5422 case VMX_VMCS_GUEST_TR_BASE:
5423 case VMX_VMCS_GUEST_GDTR_BASE:
5424 case VMX_VMCS_GUEST_IDTR_BASE:
5425 case VMX_VMCS_GUEST_RSP:
5426 case VMX_VMCS_GUEST_RIP:
5427 case VMX_VMCS_GUEST_SYSENTER_ESP:
5428 case VMX_VMCS_GUEST_SYSENTER_EIP:
5429 {
5430 if (!(u64Val >> 32))
5431 {
5432 /* If this field is 64-bit, VT-x will zero out the top bits. */
5433 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5434 }
5435 else
5436 {
5437 /* Assert that only the 32->64 switcher case should ever come here. */
5438 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5439 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5440 }
5441 break;
5442 }
5443
5444 default:
5445 {
5446 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5447 rc = VERR_INVALID_PARAMETER;
5448 break;
5449 }
5450 }
5451 AssertRCReturn(rc, rc);
5452 return rc;
5453}
5454
5455
5456/**
5457 * Queue up a VMWRITE by using the VMCS write cache.
5458 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5459 *
5460 * @param pVCpu The cross context virtual CPU structure.
5461 * @param idxField The VMCS field encoding.
5462 * @param u64Val 16, 32 or 64-bit value.
5463 */
5464VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5465{
5466 AssertPtr(pVCpu);
5467 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5468
5469 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5470 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5471
5472 /* Make sure there are no duplicates. */
5473 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5474 {
5475 if (pCache->Write.aField[i] == idxField)
5476 {
5477 pCache->Write.aFieldVal[i] = u64Val;
5478 return VINF_SUCCESS;
5479 }
5480 }
5481
5482 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5483 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5484 pCache->Write.cValidEntries++;
5485 return VINF_SUCCESS;
5486}
5487#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5488
5489
5490/**
5491 * Sets up the usage of TSC-offsetting and updates the VMCS.
5492 *
5493 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5494 * VMX preemption timer.
5495 *
5496 * @returns VBox status code.
5497 * @param pVM The cross context VM structure.
5498 * @param pVCpu The cross context virtual CPU structure.
5499 *
5500 * @remarks No-long-jump zone!!!
5501 */
5502static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5503{
5504 int rc;
5505 bool fOffsettedTsc;
5506 bool fParavirtTsc;
5507 if (pVM->hm.s.vmx.fUsePreemptTimer)
5508 {
5509 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5510 &fOffsettedTsc, &fParavirtTsc);
5511
5512 /* Make sure the returned values have sane upper and lower boundaries. */
5513 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5514 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5515 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5516 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5517
5518 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5519 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5520 }
5521 else
5522 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5523
5524 /** @todo later optimize this to be done elsewhere and not before every
5525 * VM-entry. */
5526 if (fParavirtTsc)
5527 {
5528 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5529 information before every VM-entry, hence disable it for performance sake. */
5530#if 0
5531 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5532 AssertRC(rc);
5533#endif
5534 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5535 }
5536
5537 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5538 {
5539 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5540 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5541
5542 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5543 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5544 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5545 }
5546 else
5547 {
5548 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5549 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5550 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5551 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5552 }
5553}
5554
5555
5556/**
5557 * Determines if an exception is a contributory exception.
5558 *
5559 * Contributory exceptions are ones which can cause double-faults unless the
5560 * original exception was a benign exception. Page-fault is intentionally not
5561 * included here as it's a conditional contributory exception.
5562 *
5563 * @returns true if the exception is contributory, false otherwise.
5564 * @param uVector The exception vector.
5565 */
5566DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5567{
5568 switch (uVector)
5569 {
5570 case X86_XCPT_GP:
5571 case X86_XCPT_SS:
5572 case X86_XCPT_NP:
5573 case X86_XCPT_TS:
5574 case X86_XCPT_DE:
5575 return true;
5576 default:
5577 break;
5578 }
5579 return false;
5580}
5581
5582
5583/**
5584 * Sets an event as a pending event to be injected into the guest.
5585 *
5586 * @param pVCpu The cross context virtual CPU structure.
5587 * @param u32IntInfo The VM-entry interruption-information field.
5588 * @param cbInstr The VM-entry instruction length in bytes (for software
5589 * interrupts, exceptions and privileged software
5590 * exceptions).
5591 * @param u32ErrCode The VM-entry exception error code.
5592 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5593 * page-fault.
5594 *
5595 * @remarks Statistics counter assumes this is a guest event being injected or
5596 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5597 * always incremented.
5598 */
5599DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5600 RTGCUINTPTR GCPtrFaultAddress)
5601{
5602 Assert(!pVCpu->hm.s.Event.fPending);
5603 pVCpu->hm.s.Event.fPending = true;
5604 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5605 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5606 pVCpu->hm.s.Event.cbInstr = cbInstr;
5607 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5608
5609 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5610}
5611
5612
5613/**
5614 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5615 *
5616 * @param pVCpu The cross context virtual CPU structure.
5617 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5618 * out-of-sync. Make sure to update the required fields
5619 * before using them.
5620 */
5621DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5622{
5623 NOREF(pMixedCtx);
5624 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5625 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5626 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5627 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5628}
5629
5630
5631/**
5632 * Handle a condition that occurred while delivering an event through the guest
5633 * IDT.
5634 *
5635 * @returns Strict VBox status code (i.e. informational status codes too).
5636 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5637 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5638 * to continue execution of the guest which will delivery the \#DF.
5639 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5640 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5641 *
5642 * @param pVCpu The cross context virtual CPU structure.
5643 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5644 * out-of-sync. Make sure to update the required fields
5645 * before using them.
5646 * @param pVmxTransient Pointer to the VMX transient structure.
5647 *
5648 * @remarks No-long-jump zone!!!
5649 */
5650static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5651{
5652 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5653
5654 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5655 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5656
5657 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5658 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5659 {
5660 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5661 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5662
5663 typedef enum
5664 {
5665 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5666 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5667 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5668 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5669 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5670 } VMXREFLECTXCPT;
5671
5672 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5673 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5674 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5675 {
5676 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5677 {
5678 enmReflect = VMXREFLECTXCPT_XCPT;
5679#ifdef VBOX_STRICT
5680 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5681 && uExitVector == X86_XCPT_PF)
5682 {
5683 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5684 }
5685#endif
5686 if ( uExitVector == X86_XCPT_PF
5687 && uIdtVector == X86_XCPT_PF)
5688 {
5689 pVmxTransient->fVectoringDoublePF = true;
5690 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5691 }
5692 else if ( uExitVector == X86_XCPT_AC
5693 && uIdtVector == X86_XCPT_AC)
5694 {
5695 enmReflect = VMXREFLECTXCPT_HANG;
5696 Log4(("IDT: Nested #AC - Bad guest\n"));
5697 }
5698 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5699 && hmR0VmxIsContributoryXcpt(uExitVector)
5700 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5701 || uIdtVector == X86_XCPT_PF))
5702 {
5703 enmReflect = VMXREFLECTXCPT_DF;
5704 }
5705 else if (uIdtVector == X86_XCPT_DF)
5706 enmReflect = VMXREFLECTXCPT_TF;
5707 }
5708 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5709 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5710 {
5711 /*
5712 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5713 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5714 */
5715 enmReflect = VMXREFLECTXCPT_XCPT;
5716
5717 if (uExitVector == X86_XCPT_PF)
5718 {
5719 pVmxTransient->fVectoringPF = true;
5720 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5721 }
5722 }
5723 }
5724 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5725 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5726 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5727 {
5728 /*
5729 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5730 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5731 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5732 */
5733 enmReflect = VMXREFLECTXCPT_XCPT;
5734 }
5735
5736 /*
5737 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5738 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5739 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5740 *
5741 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5742 */
5743 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5744 && enmReflect == VMXREFLECTXCPT_XCPT
5745 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5746 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5747 {
5748 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5749 }
5750
5751 switch (enmReflect)
5752 {
5753 case VMXREFLECTXCPT_XCPT:
5754 {
5755 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5756 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5757 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5758
5759 uint32_t u32ErrCode = 0;
5760 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5761 {
5762 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5763 AssertRCReturn(rc2, rc2);
5764 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5765 }
5766
5767 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5768 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5769 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5770 rcStrict = VINF_SUCCESS;
5771 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5772 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5773
5774 break;
5775 }
5776
5777 case VMXREFLECTXCPT_DF:
5778 {
5779 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5780 rcStrict = VINF_HM_DOUBLE_FAULT;
5781 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5782 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5783
5784 break;
5785 }
5786
5787 case VMXREFLECTXCPT_TF:
5788 {
5789 rcStrict = VINF_EM_RESET;
5790 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5791 uExitVector));
5792 break;
5793 }
5794
5795 case VMXREFLECTXCPT_HANG:
5796 {
5797 rcStrict = VERR_EM_GUEST_CPU_HANG;
5798 break;
5799 }
5800
5801 default:
5802 Assert(rcStrict == VINF_SUCCESS);
5803 break;
5804 }
5805 }
5806 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5807 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5808 && uExitVector != X86_XCPT_DF
5809 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5810 {
5811 /*
5812 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5813 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5814 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5815 */
5816 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5817 {
5818 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5819 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5820 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5821 }
5822 }
5823
5824 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5825 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5826 return rcStrict;
5827}
5828
5829
5830/**
5831 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5832 *
5833 * @returns VBox status code.
5834 * @param pVCpu The cross context virtual CPU structure.
5835 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5836 * out-of-sync. Make sure to update the required fields
5837 * before using them.
5838 *
5839 * @remarks No-long-jump zone!!!
5840 */
5841static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5842{
5843 NOREF(pMixedCtx);
5844
5845 /*
5846 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5847 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5848 */
5849 VMMRZCallRing3Disable(pVCpu);
5850 HM_DISABLE_PREEMPT();
5851
5852 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5853 {
5854 uint32_t uVal = 0;
5855 uint32_t uShadow = 0;
5856 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5857 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5858 AssertRCReturn(rc, rc);
5859
5860 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5861 CPUMSetGuestCR0(pVCpu, uVal);
5862 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5863 }
5864
5865 HM_RESTORE_PREEMPT();
5866 VMMRZCallRing3Enable(pVCpu);
5867 return VINF_SUCCESS;
5868}
5869
5870
5871/**
5872 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5873 *
5874 * @returns VBox status code.
5875 * @param pVCpu The cross context virtual CPU structure.
5876 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5877 * out-of-sync. Make sure to update the required fields
5878 * before using them.
5879 *
5880 * @remarks No-long-jump zone!!!
5881 */
5882static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5883{
5884 NOREF(pMixedCtx);
5885
5886 int rc = VINF_SUCCESS;
5887 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5888 {
5889 uint32_t uVal = 0;
5890 uint32_t uShadow = 0;
5891 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5892 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5893 AssertRCReturn(rc, rc);
5894
5895 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5896 CPUMSetGuestCR4(pVCpu, uVal);
5897 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5898 }
5899 return rc;
5900}
5901
5902
5903/**
5904 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5905 *
5906 * @returns VBox status code.
5907 * @param pVCpu The cross context virtual CPU structure.
5908 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5909 * out-of-sync. Make sure to update the required fields
5910 * before using them.
5911 *
5912 * @remarks No-long-jump zone!!!
5913 */
5914static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5915{
5916 int rc = VINF_SUCCESS;
5917 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5918 {
5919 uint64_t u64Val = 0;
5920 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5921 AssertRCReturn(rc, rc);
5922
5923 pMixedCtx->rip = u64Val;
5924 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5925 }
5926 return rc;
5927}
5928
5929
5930/**
5931 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5932 *
5933 * @returns VBox status code.
5934 * @param pVCpu The cross context virtual CPU structure.
5935 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5936 * out-of-sync. Make sure to update the required fields
5937 * before using them.
5938 *
5939 * @remarks No-long-jump zone!!!
5940 */
5941static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5942{
5943 int rc = VINF_SUCCESS;
5944 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5945 {
5946 uint64_t u64Val = 0;
5947 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5948 AssertRCReturn(rc, rc);
5949
5950 pMixedCtx->rsp = u64Val;
5951 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5952 }
5953 return rc;
5954}
5955
5956
5957/**
5958 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5959 *
5960 * @returns VBox status code.
5961 * @param pVCpu The cross context virtual CPU structure.
5962 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5963 * out-of-sync. Make sure to update the required fields
5964 * before using them.
5965 *
5966 * @remarks No-long-jump zone!!!
5967 */
5968static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5969{
5970 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5971 {
5972 uint32_t uVal = 0;
5973 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5974 AssertRCReturn(rc, rc);
5975
5976 pMixedCtx->eflags.u32 = uVal;
5977 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5978 {
5979 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5980 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5981
5982 pMixedCtx->eflags.Bits.u1VM = 0;
5983 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5984 }
5985
5986 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5987 }
5988 return VINF_SUCCESS;
5989}
5990
5991
5992/**
5993 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5994 * guest-CPU context.
5995 */
5996DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5997{
5998 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5999 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6000 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6001 return rc;
6002}
6003
6004
6005/**
6006 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6007 * from the guest-state area in the VMCS.
6008 *
6009 * @param pVCpu The cross context virtual CPU structure.
6010 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6011 * out-of-sync. Make sure to update the required fields
6012 * before using them.
6013 *
6014 * @remarks No-long-jump zone!!!
6015 */
6016static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6017{
6018 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6019 {
6020 uint32_t uIntrState = 0;
6021 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6022 AssertRC(rc);
6023
6024 if (!uIntrState)
6025 {
6026 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6027 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6028
6029 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6030 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6031 }
6032 else
6033 {
6034 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6035 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6036 {
6037 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6038 AssertRC(rc);
6039 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6040 AssertRC(rc);
6041
6042 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6043 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6044 }
6045 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6046 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6047
6048 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6049 {
6050 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6051 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6052 }
6053 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6054 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6055 }
6056
6057 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6058 }
6059}
6060
6061
6062/**
6063 * Saves the guest's activity state.
6064 *
6065 * @returns VBox status code.
6066 * @param pVCpu The cross context virtual CPU structure.
6067 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6068 * out-of-sync. Make sure to update the required fields
6069 * before using them.
6070 *
6071 * @remarks No-long-jump zone!!!
6072 */
6073static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6074{
6075 NOREF(pMixedCtx);
6076 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6077 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6078 return VINF_SUCCESS;
6079}
6080
6081
6082/**
6083 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6084 * the current VMCS into the guest-CPU context.
6085 *
6086 * @returns VBox status code.
6087 * @param pVCpu The cross context virtual CPU structure.
6088 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6089 * out-of-sync. Make sure to update the required fields
6090 * before using them.
6091 *
6092 * @remarks No-long-jump zone!!!
6093 */
6094static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6095{
6096 int rc = VINF_SUCCESS;
6097 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6098 {
6099 uint32_t u32Val = 0;
6100 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6101 pMixedCtx->SysEnter.cs = u32Val;
6102 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6103 }
6104
6105 uint64_t u64Val = 0;
6106 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6107 {
6108 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6109 pMixedCtx->SysEnter.eip = u64Val;
6110 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6111 }
6112 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6113 {
6114 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6115 pMixedCtx->SysEnter.esp = u64Val;
6116 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6117 }
6118 return rc;
6119}
6120
6121
6122/**
6123 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6124 * the CPU back into the guest-CPU context.
6125 *
6126 * @returns VBox status code.
6127 * @param pVCpu The cross context virtual CPU structure.
6128 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6129 * out-of-sync. Make sure to update the required fields
6130 * before using them.
6131 *
6132 * @remarks No-long-jump zone!!!
6133 */
6134static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6135{
6136#if HC_ARCH_BITS == 64
6137 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6138 {
6139 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6140 VMMRZCallRing3Disable(pVCpu);
6141 HM_DISABLE_PREEMPT();
6142
6143 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6144 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6145 {
6146 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6147 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6148 }
6149
6150 HM_RESTORE_PREEMPT();
6151 VMMRZCallRing3Enable(pVCpu);
6152 }
6153 else
6154 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6155#else
6156 NOREF(pMixedCtx);
6157 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6158#endif
6159
6160 return VINF_SUCCESS;
6161}
6162
6163
6164/**
6165 * Saves the auto load/store'd guest MSRs from the current VMCS into
6166 * the guest-CPU context.
6167 *
6168 * @returns VBox status code.
6169 * @param pVCpu The cross context virtual CPU structure.
6170 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6171 * out-of-sync. Make sure to update the required fields
6172 * before using them.
6173 *
6174 * @remarks No-long-jump zone!!!
6175 */
6176static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6177{
6178 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6179 return VINF_SUCCESS;
6180
6181 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6182 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6183 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6184 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6185 {
6186 switch (pMsr->u32Msr)
6187 {
6188 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6189 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6190 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6191 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6192 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6193 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6194 break;
6195
6196 default:
6197 {
6198 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6199 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6200 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6201 }
6202 }
6203 }
6204
6205 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6206 return VINF_SUCCESS;
6207}
6208
6209
6210/**
6211 * Saves the guest control registers from the current VMCS into the guest-CPU
6212 * context.
6213 *
6214 * @returns VBox status code.
6215 * @param pVCpu The cross context virtual CPU structure.
6216 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6217 * out-of-sync. Make sure to update the required fields
6218 * before using them.
6219 *
6220 * @remarks No-long-jump zone!!!
6221 */
6222static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6223{
6224 /* Guest CR0. Guest FPU. */
6225 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6226 AssertRCReturn(rc, rc);
6227
6228 /* Guest CR4. */
6229 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6230 AssertRCReturn(rc, rc);
6231
6232 /* Guest CR2 - updated always during the world-switch or in #PF. */
6233 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6234 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6235 {
6236 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6237 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6238
6239 PVM pVM = pVCpu->CTX_SUFF(pVM);
6240 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6241 || ( pVM->hm.s.fNestedPaging
6242 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6243 {
6244 uint64_t u64Val = 0;
6245 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6246 if (pMixedCtx->cr3 != u64Val)
6247 {
6248 CPUMSetGuestCR3(pVCpu, u64Val);
6249 if (VMMRZCallRing3IsEnabled(pVCpu))
6250 {
6251 PGMUpdateCR3(pVCpu, u64Val);
6252 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6253 }
6254 else
6255 {
6256 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6257 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6258 }
6259 }
6260
6261 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6262 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6263 {
6264 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6265 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6266 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6267 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6268 AssertRCReturn(rc, rc);
6269
6270 if (VMMRZCallRing3IsEnabled(pVCpu))
6271 {
6272 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6273 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6274 }
6275 else
6276 {
6277 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6278 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6279 }
6280 }
6281 }
6282
6283 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6284 }
6285
6286 /*
6287 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6288 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6289 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6290 *
6291 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6292 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6293 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6294 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6295 *
6296 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6297 */
6298 if (VMMRZCallRing3IsEnabled(pVCpu))
6299 {
6300 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6301 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6302
6303 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6304 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6305
6306 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6307 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6308 }
6309
6310 return rc;
6311}
6312
6313
6314/**
6315 * Reads a guest segment register from the current VMCS into the guest-CPU
6316 * context.
6317 *
6318 * @returns VBox status code.
6319 * @param pVCpu The cross context virtual CPU structure.
6320 * @param idxSel Index of the selector in the VMCS.
6321 * @param idxLimit Index of the segment limit in the VMCS.
6322 * @param idxBase Index of the segment base in the VMCS.
6323 * @param idxAccess Index of the access rights of the segment in the VMCS.
6324 * @param pSelReg Pointer to the segment selector.
6325 *
6326 * @remarks No-long-jump zone!!!
6327 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6328 * macro as that takes care of whether to read from the VMCS cache or
6329 * not.
6330 */
6331DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6332 PCPUMSELREG pSelReg)
6333{
6334 NOREF(pVCpu);
6335
6336 uint32_t u32Val = 0;
6337 int rc = VMXReadVmcs32(idxSel, &u32Val);
6338 AssertRCReturn(rc, rc);
6339 pSelReg->Sel = (uint16_t)u32Val;
6340 pSelReg->ValidSel = (uint16_t)u32Val;
6341 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6342
6343 rc = VMXReadVmcs32(idxLimit, &u32Val);
6344 AssertRCReturn(rc, rc);
6345 pSelReg->u32Limit = u32Val;
6346
6347 uint64_t u64Val = 0;
6348 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6349 AssertRCReturn(rc, rc);
6350 pSelReg->u64Base = u64Val;
6351
6352 rc = VMXReadVmcs32(idxAccess, &u32Val);
6353 AssertRCReturn(rc, rc);
6354 pSelReg->Attr.u = u32Val;
6355
6356 /*
6357 * If VT-x marks the segment as unusable, most other bits remain undefined:
6358 * - For CS the L, D and G bits have meaning.
6359 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6360 * - For the remaining data segments no bits are defined.
6361 *
6362 * The present bit and the unusable bit has been observed to be set at the
6363 * same time (the selector was supposed to be invalid as we started executing
6364 * a V8086 interrupt in ring-0).
6365 *
6366 * What should be important for the rest of the VBox code, is that the P bit is
6367 * cleared. Some of the other VBox code recognizes the unusable bit, but
6368 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6369 * safe side here, we'll strip off P and other bits we don't care about. If
6370 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6371 *
6372 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6373 */
6374 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6375 {
6376 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6377
6378 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6379 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6380 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6381
6382 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6383#ifdef DEBUG_bird
6384 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6385 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6386 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6387#endif
6388 }
6389 return VINF_SUCCESS;
6390}
6391
6392
6393#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6394# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6395 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6396 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6397#else
6398# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6399 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6400 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6401#endif
6402
6403
6404/**
6405 * Saves the guest segment registers from the current VMCS into the guest-CPU
6406 * context.
6407 *
6408 * @returns VBox status code.
6409 * @param pVCpu The cross context virtual CPU structure.
6410 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6411 * out-of-sync. Make sure to update the required fields
6412 * before using them.
6413 *
6414 * @remarks No-long-jump zone!!!
6415 */
6416static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6417{
6418 /* Guest segment registers. */
6419 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6420 {
6421 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6422 AssertRCReturn(rc, rc);
6423
6424 rc = VMXLOCAL_READ_SEG(CS, cs);
6425 rc |= VMXLOCAL_READ_SEG(SS, ss);
6426 rc |= VMXLOCAL_READ_SEG(DS, ds);
6427 rc |= VMXLOCAL_READ_SEG(ES, es);
6428 rc |= VMXLOCAL_READ_SEG(FS, fs);
6429 rc |= VMXLOCAL_READ_SEG(GS, gs);
6430 AssertRCReturn(rc, rc);
6431
6432 /* Restore segment attributes for real-on-v86 mode hack. */
6433 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6434 {
6435 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6436 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6437 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6438 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6439 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6440 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6441 }
6442 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6443 }
6444
6445 return VINF_SUCCESS;
6446}
6447
6448
6449/**
6450 * Saves the guest descriptor table registers and task register from the current
6451 * VMCS into the guest-CPU context.
6452 *
6453 * @returns VBox status code.
6454 * @param pVCpu The cross context virtual CPU structure.
6455 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6456 * out-of-sync. Make sure to update the required fields
6457 * before using them.
6458 *
6459 * @remarks No-long-jump zone!!!
6460 */
6461static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6462{
6463 int rc = VINF_SUCCESS;
6464
6465 /* Guest LDTR. */
6466 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6467 {
6468 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6469 AssertRCReturn(rc, rc);
6470 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6471 }
6472
6473 /* Guest GDTR. */
6474 uint64_t u64Val = 0;
6475 uint32_t u32Val = 0;
6476 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6477 {
6478 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6479 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6480 pMixedCtx->gdtr.pGdt = u64Val;
6481 pMixedCtx->gdtr.cbGdt = u32Val;
6482 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6483 }
6484
6485 /* Guest IDTR. */
6486 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6487 {
6488 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6489 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6490 pMixedCtx->idtr.pIdt = u64Val;
6491 pMixedCtx->idtr.cbIdt = u32Val;
6492 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6493 }
6494
6495 /* Guest TR. */
6496 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6497 {
6498 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6499 AssertRCReturn(rc, rc);
6500
6501 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6502 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6503 {
6504 rc = VMXLOCAL_READ_SEG(TR, tr);
6505 AssertRCReturn(rc, rc);
6506 }
6507 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6508 }
6509 return rc;
6510}
6511
6512#undef VMXLOCAL_READ_SEG
6513
6514
6515/**
6516 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6517 * context.
6518 *
6519 * @returns VBox status code.
6520 * @param pVCpu The cross context virtual CPU structure.
6521 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6522 * out-of-sync. Make sure to update the required fields
6523 * before using them.
6524 *
6525 * @remarks No-long-jump zone!!!
6526 */
6527static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6528{
6529 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6530 {
6531 if (!pVCpu->hm.s.fUsingHyperDR7)
6532 {
6533 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6534 uint32_t u32Val;
6535 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6536 pMixedCtx->dr[7] = u32Val;
6537 }
6538
6539 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6540 }
6541 return VINF_SUCCESS;
6542}
6543
6544
6545/**
6546 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6547 *
6548 * @returns VBox status code.
6549 * @param pVCpu The cross context virtual CPU structure.
6550 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6551 * out-of-sync. Make sure to update the required fields
6552 * before using them.
6553 *
6554 * @remarks No-long-jump zone!!!
6555 */
6556static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6557{
6558 NOREF(pMixedCtx);
6559
6560 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6561 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6562 return VINF_SUCCESS;
6563}
6564
6565
6566/**
6567 * Saves the entire guest state from the currently active VMCS into the
6568 * guest-CPU context.
6569 *
6570 * This essentially VMREADs all guest-data.
6571 *
6572 * @returns VBox status code.
6573 * @param pVCpu The cross context virtual CPU structure.
6574 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6575 * out-of-sync. Make sure to update the required fields
6576 * before using them.
6577 */
6578static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6579{
6580 Assert(pVCpu);
6581 Assert(pMixedCtx);
6582
6583 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6584 return VINF_SUCCESS;
6585
6586 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6587 again on the ring-3 callback path, there is no real need to. */
6588 if (VMMRZCallRing3IsEnabled(pVCpu))
6589 VMMR0LogFlushDisable(pVCpu);
6590 else
6591 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6592 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6593
6594 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6595 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6596
6597 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6598 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6599
6600 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6601 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6602
6603 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6604 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6605
6606 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6607 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6608
6609 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6610 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6611
6612 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6613 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6614
6615 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6616 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6617
6618 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6619 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6620
6621 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6622 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6623
6624 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6625 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6626 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6627
6628 if (VMMRZCallRing3IsEnabled(pVCpu))
6629 VMMR0LogFlushEnable(pVCpu);
6630
6631 return VINF_SUCCESS;
6632}
6633
6634
6635/**
6636 * Saves basic guest registers needed for IEM instruction execution.
6637 *
6638 * @returns VBox status code (OR-able).
6639 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6640 * @param pMixedCtx Pointer to the CPU context of the guest.
6641 * @param fMemory Whether the instruction being executed operates on
6642 * memory or not. Only CR0 is synced up if clear.
6643 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6644 */
6645static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6646{
6647 /*
6648 * We assume all general purpose registers other than RSP are available.
6649 *
6650 * RIP is a must, as it will be incremented or otherwise changed.
6651 *
6652 * RFLAGS are always required to figure the CPL.
6653 *
6654 * RSP isn't always required, however it's a GPR, so frequently required.
6655 *
6656 * SS and CS are the only segment register needed if IEM doesn't do memory
6657 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6658 *
6659 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6660 * be required for memory accesses.
6661 *
6662 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6663 */
6664 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6665 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6666 if (fNeedRsp)
6667 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6668 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6669 if (!fMemory)
6670 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6671 else
6672 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6673 AssertRCReturn(rc, rc);
6674 return rc;
6675}
6676
6677
6678/**
6679 * Ensures that we've got a complete basic guest-context.
6680 *
6681 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6682 * is for the interpreter.
6683 *
6684 * @returns VBox status code.
6685 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6686 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6687 * needing to be synced in.
6688 * @thread EMT(pVCpu)
6689 */
6690VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6691{
6692 /* Note! Since this is only applicable to VT-x, the implementation is placed
6693 in the VT-x part of the sources instead of the generic stuff. */
6694 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6695 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6696 return VINF_SUCCESS;
6697}
6698
6699
6700/**
6701 * Check per-VM and per-VCPU force flag actions that require us to go back to
6702 * ring-3 for one reason or another.
6703 *
6704 * @returns Strict VBox status code (i.e. informational status codes too)
6705 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6706 * ring-3.
6707 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6708 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6709 * interrupts)
6710 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6711 * all EMTs to be in ring-3.
6712 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6713 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6714 * to the EM loop.
6715 *
6716 * @param pVM The cross context VM structure.
6717 * @param pVCpu The cross context virtual CPU structure.
6718 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6719 * out-of-sync. Make sure to update the required fields
6720 * before using them.
6721 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6722 */
6723static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6724{
6725 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6726
6727 /*
6728 * Anything pending? Should be more likely than not if we're doing a good job.
6729 */
6730 if ( !fStepping
6731 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6732 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6733 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6734 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6735 return VINF_SUCCESS;
6736
6737 /* We need the control registers now, make sure the guest-CPU context is updated. */
6738 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6739 AssertRCReturn(rc3, rc3);
6740
6741 /* Pending HM CR3 sync. */
6742 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6743 {
6744 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6745 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6746 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6747 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6748 }
6749
6750 /* Pending HM PAE PDPEs. */
6751 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6752 {
6753 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6754 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6755 }
6756
6757 /* Pending PGM C3 sync. */
6758 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6759 {
6760 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6761 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6762 if (rcStrict2 != VINF_SUCCESS)
6763 {
6764 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6765 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6766 return rcStrict2;
6767 }
6768 }
6769
6770 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6771 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6772 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6773 {
6774 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6775 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6776 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6777 return rc2;
6778 }
6779
6780 /* Pending VM request packets, such as hardware interrupts. */
6781 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6782 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6783 {
6784 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6785 return VINF_EM_PENDING_REQUEST;
6786 }
6787
6788 /* Pending PGM pool flushes. */
6789 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6790 {
6791 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6792 return VINF_PGM_POOL_FLUSH_PENDING;
6793 }
6794
6795 /* Pending DMA requests. */
6796 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6797 {
6798 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6799 return VINF_EM_RAW_TO_R3;
6800 }
6801
6802 return VINF_SUCCESS;
6803}
6804
6805
6806/**
6807 * Converts any TRPM trap into a pending HM event. This is typically used when
6808 * entering from ring-3 (not longjmp returns).
6809 *
6810 * @param pVCpu The cross context virtual CPU structure.
6811 */
6812static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6813{
6814 Assert(TRPMHasTrap(pVCpu));
6815 Assert(!pVCpu->hm.s.Event.fPending);
6816
6817 uint8_t uVector;
6818 TRPMEVENT enmTrpmEvent;
6819 RTGCUINT uErrCode;
6820 RTGCUINTPTR GCPtrFaultAddress;
6821 uint8_t cbInstr;
6822
6823 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6824 AssertRC(rc);
6825
6826 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6827 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6828 if (enmTrpmEvent == TRPM_TRAP)
6829 {
6830 switch (uVector)
6831 {
6832 case X86_XCPT_NMI:
6833 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6834 break;
6835
6836 case X86_XCPT_BP:
6837 case X86_XCPT_OF:
6838 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6839 break;
6840
6841 case X86_XCPT_PF:
6842 case X86_XCPT_DF:
6843 case X86_XCPT_TS:
6844 case X86_XCPT_NP:
6845 case X86_XCPT_SS:
6846 case X86_XCPT_GP:
6847 case X86_XCPT_AC:
6848 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6849 /* no break! */
6850 default:
6851 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6852 break;
6853 }
6854 }
6855 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6856 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6857 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6858 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6859 else
6860 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6861
6862 rc = TRPMResetTrap(pVCpu);
6863 AssertRC(rc);
6864 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6865 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6866
6867 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6868 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6869}
6870
6871
6872/**
6873 * Converts the pending HM event into a TRPM trap.
6874 *
6875 * @param pVCpu The cross context virtual CPU structure.
6876 */
6877static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6878{
6879 Assert(pVCpu->hm.s.Event.fPending);
6880
6881 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6882 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6883 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6884 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6885
6886 /* If a trap was already pending, we did something wrong! */
6887 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6888
6889 TRPMEVENT enmTrapType;
6890 switch (uVectorType)
6891 {
6892 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6893 enmTrapType = TRPM_HARDWARE_INT;
6894 break;
6895
6896 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6897 enmTrapType = TRPM_SOFTWARE_INT;
6898 break;
6899
6900 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6901 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6902 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6903 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6904 enmTrapType = TRPM_TRAP;
6905 break;
6906
6907 default:
6908 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6909 enmTrapType = TRPM_32BIT_HACK;
6910 break;
6911 }
6912
6913 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6914
6915 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6916 AssertRC(rc);
6917
6918 if (fErrorCodeValid)
6919 TRPMSetErrorCode(pVCpu, uErrorCode);
6920
6921 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6922 && uVector == X86_XCPT_PF)
6923 {
6924 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6925 }
6926 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6927 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6928 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6929 {
6930 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6931 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6932 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6933 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6934 }
6935
6936 /* Clear any pending events from the VMCS. */
6937 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6938 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6939
6940 /* We're now done converting the pending event. */
6941 pVCpu->hm.s.Event.fPending = false;
6942}
6943
6944
6945/**
6946 * Does the necessary state syncing before returning to ring-3 for any reason
6947 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6948 *
6949 * @returns VBox status code.
6950 * @param pVM The cross context VM structure.
6951 * @param pVCpu The cross context virtual CPU structure.
6952 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6953 * be out-of-sync. Make sure to update the required
6954 * fields before using them.
6955 * @param fSaveGuestState Whether to save the guest state or not.
6956 *
6957 * @remarks No-long-jmp zone!!!
6958 */
6959static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6960{
6961 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6962 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6963
6964 RTCPUID idCpu = RTMpCpuId();
6965 Log4Func(("HostCpuId=%u\n", idCpu));
6966
6967 /*
6968 * !!! IMPORTANT !!!
6969 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6970 */
6971
6972 /* Save the guest state if necessary. */
6973 if ( fSaveGuestState
6974 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6975 {
6976 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6977 AssertRCReturn(rc, rc);
6978 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6979 }
6980
6981 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6982 if (CPUMIsGuestFPUStateActive(pVCpu))
6983 {
6984 /* We shouldn't reload CR0 without saving it first. */
6985 if (!fSaveGuestState)
6986 {
6987 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6988 AssertRCReturn(rc, rc);
6989 }
6990 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6991 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6992 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6993 }
6994
6995 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6996#ifdef VBOX_STRICT
6997 if (CPUMIsHyperDebugStateActive(pVCpu))
6998 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6999#endif
7000 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7001 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7002 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7003 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7004
7005#if HC_ARCH_BITS == 64
7006 /* Restore host-state bits that VT-x only restores partially. */
7007 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7008 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7009 {
7010 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7011 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7012 }
7013 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7014#endif
7015
7016#if HC_ARCH_BITS == 64
7017 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7018 if ( pVM->hm.s.fAllow64BitGuests
7019 && pVCpu->hm.s.vmx.fLazyMsrs)
7020 {
7021 /* We shouldn't reload the guest MSRs without saving it first. */
7022 if (!fSaveGuestState)
7023 {
7024 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7025 AssertRCReturn(rc, rc);
7026 }
7027 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7028 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7029 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7030 }
7031#endif
7032
7033 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7034 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7035
7036 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7037 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7038 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7039 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7040 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7041 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7042 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7043 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7044
7045 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7046
7047 /** @todo This partially defeats the purpose of having preemption hooks.
7048 * The problem is, deregistering the hooks should be moved to a place that
7049 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7050 * context.
7051 */
7052 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7053 {
7054 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7055 AssertRCReturn(rc, rc);
7056
7057 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7058 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7059 }
7060 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7061 NOREF(idCpu);
7062
7063 return VINF_SUCCESS;
7064}
7065
7066
7067/**
7068 * Leaves the VT-x session.
7069 *
7070 * @returns VBox status code.
7071 * @param pVM The cross context VM structure.
7072 * @param pVCpu The cross context virtual CPU structure.
7073 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7074 * out-of-sync. Make sure to update the required fields
7075 * before using them.
7076 *
7077 * @remarks No-long-jmp zone!!!
7078 */
7079DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7080{
7081 HM_DISABLE_PREEMPT();
7082 HMVMX_ASSERT_CPU_SAFE();
7083 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7084 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7085
7086 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7087 and done this from the VMXR0ThreadCtxCallback(). */
7088 if (!pVCpu->hm.s.fLeaveDone)
7089 {
7090 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7091 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7092 pVCpu->hm.s.fLeaveDone = true;
7093 }
7094 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7095
7096 /*
7097 * !!! IMPORTANT !!!
7098 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7099 */
7100
7101 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7102 /** @todo Deregistering here means we need to VMCLEAR always
7103 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7104 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7105 VMMR0ThreadCtxHookDisable(pVCpu);
7106
7107 /* Leave HM context. This takes care of local init (term). */
7108 int rc = HMR0LeaveCpu(pVCpu);
7109
7110 HM_RESTORE_PREEMPT();
7111 return rc;
7112}
7113
7114
7115/**
7116 * Does the necessary state syncing before doing a longjmp to ring-3.
7117 *
7118 * @returns VBox status code.
7119 * @param pVM The cross context VM structure.
7120 * @param pVCpu The cross context virtual CPU structure.
7121 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7122 * out-of-sync. Make sure to update the required fields
7123 * before using them.
7124 *
7125 * @remarks No-long-jmp zone!!!
7126 */
7127DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7128{
7129 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7130}
7131
7132
7133/**
7134 * Take necessary actions before going back to ring-3.
7135 *
7136 * An action requires us to go back to ring-3. This function does the necessary
7137 * steps before we can safely return to ring-3. This is not the same as longjmps
7138 * to ring-3, this is voluntary and prepares the guest so it may continue
7139 * executing outside HM (recompiler/IEM).
7140 *
7141 * @returns VBox status code.
7142 * @param pVM The cross context VM structure.
7143 * @param pVCpu The cross context virtual CPU structure.
7144 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7145 * out-of-sync. Make sure to update the required fields
7146 * before using them.
7147 * @param rcExit The reason for exiting to ring-3. Can be
7148 * VINF_VMM_UNKNOWN_RING3_CALL.
7149 */
7150static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7151{
7152 Assert(pVM);
7153 Assert(pVCpu);
7154 Assert(pMixedCtx);
7155 HMVMX_ASSERT_PREEMPT_SAFE();
7156
7157 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7158 {
7159 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7160 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7161 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7162 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7163 }
7164
7165 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7166 VMMRZCallRing3Disable(pVCpu);
7167 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7168
7169 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7170 if (pVCpu->hm.s.Event.fPending)
7171 {
7172 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7173 Assert(!pVCpu->hm.s.Event.fPending);
7174 }
7175
7176 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7177 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7178
7179 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7180 and if we're injecting an event we should have a TRPM trap pending. */
7181 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7182#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7183 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7184#endif
7185
7186 /* Save guest state and restore host state bits. */
7187 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7188 AssertRCReturn(rc, rc);
7189 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7190 /* Thread-context hooks are unregistered at this point!!! */
7191
7192 /* Sync recompiler state. */
7193 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7194 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7195 | CPUM_CHANGED_LDTR
7196 | CPUM_CHANGED_GDTR
7197 | CPUM_CHANGED_IDTR
7198 | CPUM_CHANGED_TR
7199 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7200 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7201 if ( pVM->hm.s.fNestedPaging
7202 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7203 {
7204 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7205 }
7206
7207 Assert(!pVCpu->hm.s.fClearTrapFlag);
7208
7209 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7210 if (rcExit != VINF_EM_RAW_INTERRUPT)
7211 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7212
7213 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7214
7215 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7216 VMMRZCallRing3RemoveNotification(pVCpu);
7217 VMMRZCallRing3Enable(pVCpu);
7218
7219 return rc;
7220}
7221
7222
7223/**
7224 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7225 * longjump to ring-3 and possibly get preempted.
7226 *
7227 * @returns VBox status code.
7228 * @param pVCpu The cross context virtual CPU structure.
7229 * @param enmOperation The operation causing the ring-3 longjump.
7230 * @param pvUser Opaque pointer to the guest-CPU context. The data
7231 * may be out-of-sync. Make sure to update the required
7232 * fields before using them.
7233 */
7234static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7235{
7236 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7237 {
7238 /*
7239 * !!! IMPORTANT !!!
7240 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7241 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7242 */
7243 VMMRZCallRing3RemoveNotification(pVCpu);
7244 VMMRZCallRing3Disable(pVCpu);
7245 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7246 RTThreadPreemptDisable(&PreemptState);
7247
7248 PVM pVM = pVCpu->CTX_SUFF(pVM);
7249 if (CPUMIsGuestFPUStateActive(pVCpu))
7250 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7251
7252 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7253
7254#if HC_ARCH_BITS == 64
7255 /* Restore host-state bits that VT-x only restores partially. */
7256 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7257 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7258 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7259 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7260
7261 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7262 if ( pVM->hm.s.fAllow64BitGuests
7263 && pVCpu->hm.s.vmx.fLazyMsrs)
7264 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7265#endif
7266 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7267 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7268 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7269 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7270 {
7271 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7272 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7273 }
7274
7275 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7276 VMMR0ThreadCtxHookDisable(pVCpu);
7277 HMR0LeaveCpu(pVCpu);
7278 RTThreadPreemptRestore(&PreemptState);
7279 return VINF_SUCCESS;
7280 }
7281
7282 Assert(pVCpu);
7283 Assert(pvUser);
7284 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7285 HMVMX_ASSERT_PREEMPT_SAFE();
7286
7287 VMMRZCallRing3Disable(pVCpu);
7288 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7289
7290 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7291 enmOperation));
7292
7293 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7294 AssertRCReturn(rc, rc);
7295
7296 VMMRZCallRing3Enable(pVCpu);
7297 return VINF_SUCCESS;
7298}
7299
7300
7301/**
7302 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7303 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7304 *
7305 * @param pVCpu The cross context virtual CPU structure.
7306 */
7307DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7308{
7309 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7310 {
7311 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7312 {
7313 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7314 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7315 AssertRC(rc);
7316 Log4(("Setup interrupt-window exiting\n"));
7317 }
7318 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7319}
7320
7321
7322/**
7323 * Clears the interrupt-window exiting control in the VMCS.
7324 *
7325 * @param pVCpu The cross context virtual CPU structure.
7326 */
7327DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7328{
7329 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7330 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7331 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7332 AssertRC(rc);
7333 Log4(("Cleared interrupt-window exiting\n"));
7334}
7335
7336
7337/**
7338 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7339 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7340 *
7341 * @param pVCpu The cross context virtual CPU structure.
7342 */
7343DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7344{
7345 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7346 {
7347 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7348 {
7349 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7350 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7351 AssertRC(rc);
7352 Log4(("Setup NMI-window exiting\n"));
7353 }
7354 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7355}
7356
7357
7358/**
7359 * Clears the NMI-window exiting control in the VMCS.
7360 *
7361 * @param pVCpu The cross context virtual CPU structure.
7362 */
7363DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7364{
7365 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7366 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7367 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7368 AssertRC(rc);
7369 Log4(("Cleared NMI-window exiting\n"));
7370}
7371
7372
7373/**
7374 * Evaluates the event to be delivered to the guest and sets it as the pending
7375 * event.
7376 *
7377 * @param pVCpu The cross context virtual CPU structure.
7378 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7379 * out-of-sync. Make sure to update the required fields
7380 * before using them.
7381 */
7382static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7383{
7384 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7385 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7386 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7387 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7388 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7389
7390 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7391 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7392 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7393 Assert(!TRPMHasTrap(pVCpu));
7394
7395 /*
7396 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7397 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7398 */
7399 /** @todo SMI. SMIs take priority over NMIs. */
7400 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7401 {
7402 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7403 if ( !pVCpu->hm.s.Event.fPending
7404 && !fBlockNmi
7405 && !fBlockSti
7406 && !fBlockMovSS)
7407 {
7408 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7409 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7410 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7411
7412 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7413 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7414 }
7415 else
7416 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7417 }
7418 /*
7419 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7420 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7421 */
7422 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7423 && !pVCpu->hm.s.fSingleInstruction)
7424 {
7425 Assert(!DBGFIsStepping(pVCpu));
7426 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7427 AssertRC(rc);
7428 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7429 if ( !pVCpu->hm.s.Event.fPending
7430 && !fBlockInt
7431 && !fBlockSti
7432 && !fBlockMovSS)
7433 {
7434 uint8_t u8Interrupt;
7435 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7436 if (RT_SUCCESS(rc))
7437 {
7438 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7439 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7440 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7441
7442 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7443 }
7444 else
7445 {
7446 /** @todo Does this actually happen? If not turn it into an assertion. */
7447 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7448 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7449 }
7450 }
7451 else
7452 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7453 }
7454}
7455
7456
7457/**
7458 * Sets a pending-debug exception to be delivered to the guest if the guest is
7459 * single-stepping in the VMCS.
7460 *
7461 * @param pVCpu The cross context virtual CPU structure.
7462 */
7463DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7464{
7465 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7466 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7467 AssertRC(rc);
7468}
7469
7470
7471/**
7472 * Injects any pending events into the guest if the guest is in a state to
7473 * receive them.
7474 *
7475 * @returns Strict VBox status code (i.e. informational status codes too).
7476 * @param pVCpu The cross context virtual CPU structure.
7477 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7478 * out-of-sync. Make sure to update the required fields
7479 * before using them.
7480 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7481 * return VINF_EM_DBG_STEPPED if the event was
7482 * dispatched directly.
7483 */
7484static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7485{
7486 HMVMX_ASSERT_PREEMPT_SAFE();
7487 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7488
7489 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7490 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7491 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7492 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7493
7494 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7495 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7496 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7497 Assert(!TRPMHasTrap(pVCpu));
7498
7499 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7500 if (pVCpu->hm.s.Event.fPending)
7501 {
7502 /*
7503 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7504 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7505 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7506 *
7507 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7508 */
7509 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7510#ifdef VBOX_STRICT
7511 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7512 {
7513 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7514 Assert(!fBlockInt);
7515 Assert(!fBlockSti);
7516 Assert(!fBlockMovSS);
7517 }
7518 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7519 {
7520 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7521 Assert(!fBlockSti);
7522 Assert(!fBlockMovSS);
7523 Assert(!fBlockNmi);
7524 }
7525#endif
7526 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7527 (uint8_t)uIntType));
7528 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7529 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7530 fStepping, &uIntrState);
7531 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7532
7533 /* Update the interruptibility-state as it could have been changed by
7534 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7535 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7536 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7537
7538 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7539 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7540 else
7541 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7542 }
7543
7544 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7545 if ( fBlockSti
7546 || fBlockMovSS)
7547 {
7548 if (!pVCpu->hm.s.fSingleInstruction)
7549 {
7550 /*
7551 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7552 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7553 * See Intel spec. 27.3.4 "Saving Non-Register State".
7554 */
7555 Assert(!DBGFIsStepping(pVCpu));
7556 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7557 AssertRCReturn(rc2, rc2);
7558 if (pMixedCtx->eflags.Bits.u1TF)
7559 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7560 }
7561 else if (pMixedCtx->eflags.Bits.u1TF)
7562 {
7563 /*
7564 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7565 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7566 */
7567 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7568 uIntrState = 0;
7569 }
7570 }
7571
7572 /*
7573 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7574 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7575 */
7576 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7577 AssertRC(rc2);
7578
7579 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7580 NOREF(fBlockMovSS); NOREF(fBlockSti);
7581 return rcStrict;
7582}
7583
7584
7585/**
7586 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7587 *
7588 * @param pVCpu The cross context virtual CPU structure.
7589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7590 * out-of-sync. Make sure to update the required fields
7591 * before using them.
7592 */
7593DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7594{
7595 NOREF(pMixedCtx);
7596 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7597 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7598}
7599
7600
7601/**
7602 * Injects a double-fault (\#DF) exception into the VM.
7603 *
7604 * @returns Strict VBox status code (i.e. informational status codes too).
7605 * @param pVCpu The cross context virtual CPU structure.
7606 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7607 * out-of-sync. Make sure to update the required fields
7608 * before using them.
7609 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7610 * and should return VINF_EM_DBG_STEPPED if the event
7611 * is injected directly (register modified by us, not
7612 * by hardware on VM-entry).
7613 * @param puIntrState Pointer to the current guest interruptibility-state.
7614 * This interruptibility-state will be updated if
7615 * necessary. This cannot not be NULL.
7616 */
7617DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7618{
7619 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7620 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7621 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7622 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7623 fStepping, puIntrState);
7624}
7625
7626
7627/**
7628 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7629 *
7630 * @param pVCpu The cross context virtual CPU structure.
7631 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7632 * out-of-sync. Make sure to update the required fields
7633 * before using them.
7634 */
7635DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7636{
7637 NOREF(pMixedCtx);
7638 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7639 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7640 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7641}
7642
7643
7644/**
7645 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7646 *
7647 * @param pVCpu The cross context virtual CPU structure.
7648 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7649 * out-of-sync. Make sure to update the required fields
7650 * before using them.
7651 * @param cbInstr The value of RIP that is to be pushed on the guest
7652 * stack.
7653 */
7654DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7655{
7656 NOREF(pMixedCtx);
7657 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7658 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7659 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7660}
7661
7662
7663/**
7664 * Injects a general-protection (\#GP) fault into the VM.
7665 *
7666 * @returns Strict VBox status code (i.e. informational status codes too).
7667 * @param pVCpu The cross context virtual CPU structure.
7668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7669 * out-of-sync. Make sure to update the required fields
7670 * before using them.
7671 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7672 * mode, i.e. in real-mode it's not valid).
7673 * @param u32ErrorCode The error code associated with the \#GP.
7674 * @param fStepping Whether we're running in
7675 * hmR0VmxRunGuestCodeStep() and should return
7676 * VINF_EM_DBG_STEPPED if the event is injected
7677 * directly (register modified by us, not by
7678 * hardware on VM-entry).
7679 * @param puIntrState Pointer to the current guest interruptibility-state.
7680 * This interruptibility-state will be updated if
7681 * necessary. This cannot not be NULL.
7682 */
7683DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7684 bool fStepping, uint32_t *puIntrState)
7685{
7686 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7687 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7688 if (fErrorCodeValid)
7689 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7690 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7691 fStepping, puIntrState);
7692}
7693
7694
7695/**
7696 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7697 * VM.
7698 *
7699 * @param pVCpu The cross context virtual CPU structure.
7700 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7701 * out-of-sync. Make sure to update the required fields
7702 * before using them.
7703 * @param u32ErrorCode The error code associated with the \#GP.
7704 */
7705DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7706{
7707 NOREF(pMixedCtx);
7708 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7709 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7710 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7711 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7712}
7713
7714
7715/**
7716 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7717 *
7718 * @param pVCpu The cross context virtual CPU structure.
7719 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7720 * out-of-sync. Make sure to update the required fields
7721 * before using them.
7722 * @param uVector The software interrupt vector number.
7723 * @param cbInstr The value of RIP that is to be pushed on the guest
7724 * stack.
7725 */
7726DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7727{
7728 NOREF(pMixedCtx);
7729 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7730 if ( uVector == X86_XCPT_BP
7731 || uVector == X86_XCPT_OF)
7732 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7733 else
7734 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7735 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7736}
7737
7738
7739/**
7740 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7741 * stack.
7742 *
7743 * @returns Strict VBox status code (i.e. informational status codes too).
7744 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7745 * @param pVM The cross context VM structure.
7746 * @param pMixedCtx Pointer to the guest-CPU context.
7747 * @param uValue The value to push to the guest stack.
7748 */
7749DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7750{
7751 /*
7752 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7753 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7754 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7755 */
7756 if (pMixedCtx->sp == 1)
7757 return VINF_EM_RESET;
7758 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7759 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7760 AssertRC(rc);
7761 return rc;
7762}
7763
7764
7765/**
7766 * Injects an event into the guest upon VM-entry by updating the relevant fields
7767 * in the VM-entry area in the VMCS.
7768 *
7769 * @returns Strict VBox status code (i.e. informational status codes too).
7770 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7771 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7772 *
7773 * @param pVCpu The cross context virtual CPU structure.
7774 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7775 * be out-of-sync. Make sure to update the required
7776 * fields before using them.
7777 * @param u64IntInfo The VM-entry interruption-information field.
7778 * @param cbInstr The VM-entry instruction length in bytes (for
7779 * software interrupts, exceptions and privileged
7780 * software exceptions).
7781 * @param u32ErrCode The VM-entry exception error code.
7782 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7783 * @param puIntrState Pointer to the current guest interruptibility-state.
7784 * This interruptibility-state will be updated if
7785 * necessary. This cannot not be NULL.
7786 * @param fStepping Whether we're running in
7787 * hmR0VmxRunGuestCodeStep() and should return
7788 * VINF_EM_DBG_STEPPED if the event is injected
7789 * directly (register modified by us, not by
7790 * hardware on VM-entry).
7791 *
7792 * @remarks Requires CR0!
7793 * @remarks No-long-jump zone!!!
7794 */
7795static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7796 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7797 uint32_t *puIntrState)
7798{
7799 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7800 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7801 Assert(puIntrState);
7802 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7803
7804 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7805 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7806
7807#ifdef VBOX_STRICT
7808 /* Validate the error-code-valid bit for hardware exceptions. */
7809 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7810 {
7811 switch (uVector)
7812 {
7813 case X86_XCPT_PF:
7814 case X86_XCPT_DF:
7815 case X86_XCPT_TS:
7816 case X86_XCPT_NP:
7817 case X86_XCPT_SS:
7818 case X86_XCPT_GP:
7819 case X86_XCPT_AC:
7820 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7821 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7822 /* fallthru */
7823 default:
7824 break;
7825 }
7826 }
7827#endif
7828
7829 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7830 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7831 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7832
7833 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7834
7835 /* We require CR0 to check if the guest is in real-mode. */
7836 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7837 AssertRCReturn(rc, rc);
7838
7839 /*
7840 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7841 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7842 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7843 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7844 */
7845 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7846 {
7847 PVM pVM = pVCpu->CTX_SUFF(pVM);
7848 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7849 {
7850 Assert(PDMVmmDevHeapIsEnabled(pVM));
7851 Assert(pVM->hm.s.vmx.pRealModeTSS);
7852
7853 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7854 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7855 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7856 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7857 AssertRCReturn(rc, rc);
7858 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7859
7860 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7861 size_t const cbIdtEntry = sizeof(X86IDTR16);
7862 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7863 {
7864 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7865 if (uVector == X86_XCPT_DF)
7866 return VINF_EM_RESET;
7867
7868 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7869 if (uVector == X86_XCPT_GP)
7870 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7871
7872 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7873 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7874 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7875 fStepping, puIntrState);
7876 }
7877
7878 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7879 uint16_t uGuestIp = pMixedCtx->ip;
7880 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7881 {
7882 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7883 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7884 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7885 }
7886 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7887 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7888
7889 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7890 X86IDTR16 IdtEntry;
7891 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7892 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7893 AssertRCReturn(rc, rc);
7894
7895 /* Construct the stack frame for the interrupt/exception handler. */
7896 VBOXSTRICTRC rcStrict;
7897 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7898 if (rcStrict == VINF_SUCCESS)
7899 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7900 if (rcStrict == VINF_SUCCESS)
7901 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7902
7903 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7904 if (rcStrict == VINF_SUCCESS)
7905 {
7906 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7907 pMixedCtx->rip = IdtEntry.offSel;
7908 pMixedCtx->cs.Sel = IdtEntry.uSel;
7909 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7910 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7911 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7912 && uVector == X86_XCPT_PF)
7913 pMixedCtx->cr2 = GCPtrFaultAddress;
7914
7915 /* If any other guest-state bits are changed here, make sure to update
7916 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7917 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7918 | HM_CHANGED_GUEST_RIP
7919 | HM_CHANGED_GUEST_RFLAGS
7920 | HM_CHANGED_GUEST_RSP);
7921
7922 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7923 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7924 {
7925 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7926 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7927 Log4(("Clearing inhibition due to STI.\n"));
7928 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7929 }
7930 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7931 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7932
7933 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7934 it, if we are returning to ring-3 before executing guest code. */
7935 pVCpu->hm.s.Event.fPending = false;
7936
7937 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7938 if (fStepping)
7939 rcStrict = VINF_EM_DBG_STEPPED;
7940 }
7941 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7942 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7943 return rcStrict;
7944 }
7945
7946 /*
7947 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7948 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7949 */
7950 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7951 }
7952
7953 /* Validate. */
7954 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7955 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7956 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7957
7958 /* Inject. */
7959 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7960 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7961 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7962 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7963
7964 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7965 && uVector == X86_XCPT_PF)
7966 pMixedCtx->cr2 = GCPtrFaultAddress;
7967
7968 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7969 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7970
7971 AssertRCReturn(rc, rc);
7972 return VINF_SUCCESS;
7973}
7974
7975
7976/**
7977 * Clears the interrupt-window exiting control in the VMCS and if necessary
7978 * clears the current event in the VMCS as well.
7979 *
7980 * @returns VBox status code.
7981 * @param pVCpu The cross context virtual CPU structure.
7982 *
7983 * @remarks Use this function only to clear events that have not yet been
7984 * delivered to the guest but are injected in the VMCS!
7985 * @remarks No-long-jump zone!!!
7986 */
7987static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7988{
7989 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7990
7991 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7992 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7993
7994 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7995 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7996}
7997
7998
7999/**
8000 * Enters the VT-x session.
8001 *
8002 * @returns VBox status code.
8003 * @param pVM The cross context VM structure.
8004 * @param pVCpu The cross context virtual CPU structure.
8005 * @param pCpu Pointer to the CPU info struct.
8006 */
8007VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8008{
8009 AssertPtr(pVM);
8010 AssertPtr(pVCpu);
8011 Assert(pVM->hm.s.vmx.fSupported);
8012 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8013 NOREF(pCpu); NOREF(pVM);
8014
8015 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8016 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8017
8018#ifdef VBOX_STRICT
8019 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8020 RTCCUINTREG uHostCR4 = ASMGetCR4();
8021 if (!(uHostCR4 & X86_CR4_VMXE))
8022 {
8023 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8024 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8025 }
8026#endif
8027
8028 /*
8029 * Load the VCPU's VMCS as the current (and active) one.
8030 */
8031 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8032 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8033 if (RT_FAILURE(rc))
8034 return rc;
8035
8036 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8037 pVCpu->hm.s.fLeaveDone = false;
8038 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8039
8040 return VINF_SUCCESS;
8041}
8042
8043
8044/**
8045 * The thread-context callback (only on platforms which support it).
8046 *
8047 * @param enmEvent The thread-context event.
8048 * @param pVCpu The cross context virtual CPU structure.
8049 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8050 * @thread EMT(pVCpu)
8051 */
8052VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8053{
8054 NOREF(fGlobalInit);
8055
8056 switch (enmEvent)
8057 {
8058 case RTTHREADCTXEVENT_OUT:
8059 {
8060 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8061 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8062 VMCPU_ASSERT_EMT(pVCpu);
8063
8064 PVM pVM = pVCpu->CTX_SUFF(pVM);
8065 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8066
8067 /* No longjmps (logger flushes, locks) in this fragile context. */
8068 VMMRZCallRing3Disable(pVCpu);
8069 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8070
8071 /*
8072 * Restore host-state (FPU, debug etc.)
8073 */
8074 if (!pVCpu->hm.s.fLeaveDone)
8075 {
8076 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8077 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8078 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8079 pVCpu->hm.s.fLeaveDone = true;
8080 }
8081
8082 /* Leave HM context, takes care of local init (term). */
8083 int rc = HMR0LeaveCpu(pVCpu);
8084 AssertRC(rc); NOREF(rc);
8085
8086 /* Restore longjmp state. */
8087 VMMRZCallRing3Enable(pVCpu);
8088 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8089 break;
8090 }
8091
8092 case RTTHREADCTXEVENT_IN:
8093 {
8094 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8095 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8096 VMCPU_ASSERT_EMT(pVCpu);
8097
8098 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8099 VMMRZCallRing3Disable(pVCpu);
8100 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8101
8102 /* Initialize the bare minimum state required for HM. This takes care of
8103 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8104 int rc = HMR0EnterCpu(pVCpu);
8105 AssertRC(rc);
8106 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8107
8108 /* Load the active VMCS as the current one. */
8109 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8110 {
8111 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8112 AssertRC(rc); NOREF(rc);
8113 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8114 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8115 }
8116 pVCpu->hm.s.fLeaveDone = false;
8117
8118 /* Restore longjmp state. */
8119 VMMRZCallRing3Enable(pVCpu);
8120 break;
8121 }
8122
8123 default:
8124 break;
8125 }
8126}
8127
8128
8129/**
8130 * Saves the host state in the VMCS host-state.
8131 * Sets up the VM-exit MSR-load area.
8132 *
8133 * The CPU state will be loaded from these fields on every successful VM-exit.
8134 *
8135 * @returns VBox status code.
8136 * @param pVM The cross context VM structure.
8137 * @param pVCpu The cross context virtual CPU structure.
8138 *
8139 * @remarks No-long-jump zone!!!
8140 */
8141static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8142{
8143 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8144
8145 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8146 return VINF_SUCCESS;
8147
8148 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8149 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8150
8151 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8152 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8153
8154 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8155 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8156
8157 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8158 return rc;
8159}
8160
8161
8162/**
8163 * Saves the host state in the VMCS host-state.
8164 *
8165 * @returns VBox status code.
8166 * @param pVM The cross context VM structure.
8167 * @param pVCpu The cross context virtual CPU structure.
8168 *
8169 * @remarks No-long-jump zone!!!
8170 */
8171VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8172{
8173 AssertPtr(pVM);
8174 AssertPtr(pVCpu);
8175
8176 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8177
8178 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8179 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8180 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8181 return hmR0VmxSaveHostState(pVM, pVCpu);
8182}
8183
8184
8185/**
8186 * Loads the guest state into the VMCS guest-state area.
8187 *
8188 * The will typically be done before VM-entry when the guest-CPU state and the
8189 * VMCS state may potentially be out of sync.
8190 *
8191 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8192 * VM-entry controls.
8193 * Sets up the appropriate VMX non-root function to execute guest code based on
8194 * the guest CPU mode.
8195 *
8196 * @returns VBox status code.
8197 * @param pVM The cross context VM structure.
8198 * @param pVCpu The cross context virtual CPU structure.
8199 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8200 * out-of-sync. Make sure to update the required fields
8201 * before using them.
8202 *
8203 * @remarks No-long-jump zone!!!
8204 */
8205static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8206{
8207 AssertPtr(pVM);
8208 AssertPtr(pVCpu);
8209 AssertPtr(pMixedCtx);
8210 HMVMX_ASSERT_PREEMPT_SAFE();
8211
8212 VMMRZCallRing3Disable(pVCpu);
8213 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8214
8215 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8216
8217 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8218
8219 /* Determine real-on-v86 mode. */
8220 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8221 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8222 && CPUMIsGuestInRealModeEx(pMixedCtx))
8223 {
8224 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8225 }
8226
8227 /*
8228 * Load the guest-state into the VMCS.
8229 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8230 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8231 */
8232 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8233 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8234
8235 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8236 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8237 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8238
8239 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8240 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8241 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8242
8243 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8244 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8245
8246 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8247 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8248
8249 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8250 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8251 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8252
8253 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8254 determine we don't have to swap EFER after all. */
8255 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8256 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8257
8258 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8259 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8260
8261 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8262 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8263
8264 /*
8265 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8266 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8267 */
8268 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8269 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8270
8271 /* Clear any unused and reserved bits. */
8272 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8273
8274 VMMRZCallRing3Enable(pVCpu);
8275
8276 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8277 return rc;
8278}
8279
8280
8281/**
8282 * Loads the state shared between the host and guest into the VMCS.
8283 *
8284 * @param pVM The cross context VM structure.
8285 * @param pVCpu The cross context virtual CPU structure.
8286 * @param pCtx Pointer to the guest-CPU context.
8287 *
8288 * @remarks No-long-jump zone!!!
8289 */
8290static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8291{
8292 NOREF(pVM);
8293
8294 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8295 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8296
8297 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8298 {
8299 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8300 AssertRC(rc);
8301 }
8302
8303 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8304 {
8305 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8306 AssertRC(rc);
8307
8308 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8309 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8310 {
8311 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8312 AssertRC(rc);
8313 }
8314 }
8315
8316 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8317 {
8318#if HC_ARCH_BITS == 64
8319 if (pVM->hm.s.fAllow64BitGuests)
8320 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8321#endif
8322 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8323 }
8324
8325 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8326 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8327 {
8328 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8329 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8330 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8331 AssertRC(rc);
8332 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8333 }
8334
8335 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8336 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8337}
8338
8339
8340/**
8341 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8342 *
8343 * @returns Strict VBox status code (i.e. informational status codes too).
8344 * @param pVM The cross context VM structure.
8345 * @param pVCpu The cross context virtual CPU structure.
8346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8347 * out-of-sync. Make sure to update the required fields
8348 * before using them.
8349 */
8350static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8351{
8352 HMVMX_ASSERT_PREEMPT_SAFE();
8353
8354 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8355#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8356 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8357#endif
8358
8359 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8360 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8361 {
8362 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8363 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8364 { /* likely */}
8365 else
8366 {
8367 AssertLogRelMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8368 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8369 }
8370 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8371 }
8372 else if (HMCPU_CF_VALUE(pVCpu))
8373 {
8374 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8375 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8376 { /* likely */}
8377 else
8378 {
8379 AssertLogRelMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n",
8380 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8381 }
8382 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8383 }
8384
8385 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8386 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8387 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8388 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8389 return rcStrict;
8390}
8391
8392
8393/**
8394 * Does the preparations before executing guest code in VT-x.
8395 *
8396 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8397 * recompiler/IEM. We must be cautious what we do here regarding committing
8398 * guest-state information into the VMCS assuming we assuredly execute the
8399 * guest in VT-x mode.
8400 *
8401 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8402 * the common-state (TRPM/forceflags), we must undo those changes so that the
8403 * recompiler/IEM can (and should) use them when it resumes guest execution.
8404 * Otherwise such operations must be done when we can no longer exit to ring-3.
8405 *
8406 * @returns Strict VBox status code (i.e. informational status codes too).
8407 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8408 * have been disabled.
8409 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8410 * double-fault into the guest.
8411 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8412 * dispatched directly.
8413 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8414 *
8415 * @param pVM The cross context VM structure.
8416 * @param pVCpu The cross context virtual CPU structure.
8417 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8418 * out-of-sync. Make sure to update the required fields
8419 * before using them.
8420 * @param pVmxTransient Pointer to the VMX transient structure.
8421 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8422 * us ignore some of the reasons for returning to
8423 * ring-3, and return VINF_EM_DBG_STEPPED if event
8424 * dispatching took place.
8425 */
8426static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8427{
8428 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8429
8430#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8431 PGMRZDynMapFlushAutoSet(pVCpu);
8432#endif
8433
8434 /* Check force flag actions that might require us to go back to ring-3. */
8435 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8436 if (rcStrict == VINF_SUCCESS)
8437 { /* FFs doesn't get set all the time. */ }
8438 else
8439 return rcStrict;
8440
8441#ifndef IEM_VERIFICATION_MODE_FULL
8442 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8443 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8444 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8445 {
8446 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8447 RTGCPHYS GCPhysApicBase;
8448 GCPhysApicBase = pMixedCtx->msrApicBase;
8449 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8450
8451 /* Unalias any existing mapping. */
8452 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8453 AssertRCReturn(rc, rc);
8454
8455 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8456 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8457 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8458 AssertRCReturn(rc, rc);
8459
8460 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8461 }
8462#endif /* !IEM_VERIFICATION_MODE_FULL */
8463
8464 if (TRPMHasTrap(pVCpu))
8465 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8466 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8467
8468 /*
8469 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8470 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8471 */
8472 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8473 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8474 { /* likely */ }
8475 else
8476 {
8477 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8478 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8479 return rcStrict;
8480 }
8481
8482 /*
8483 * Load the guest state bits, we can handle longjmps/getting preempted here.
8484 *
8485 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8486 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8487 * Hence, this needs to be done -after- injection of events.
8488 */
8489 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8490 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8491 { /* likely */ }
8492 else
8493 return rcStrict;
8494
8495 /*
8496 * No longjmps to ring-3 from this point on!!!
8497 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8498 * This also disables flushing of the R0-logger instance (if any).
8499 */
8500 VMMRZCallRing3Disable(pVCpu);
8501
8502 /*
8503 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8504 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8505 *
8506 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8507 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8508 *
8509 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8510 * executing guest code.
8511 */
8512 pVmxTransient->fEFlags = ASMIntDisableFlags();
8513
8514 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8515 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8516 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8517 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8518 {
8519 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8520 {
8521 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8522 pVCpu->hm.s.Event.fPending = false;
8523
8524 return VINF_SUCCESS;
8525 }
8526
8527 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8528 rcStrict = VINF_EM_RAW_INTERRUPT;
8529 }
8530 else
8531 {
8532 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8533 rcStrict = VINF_EM_RAW_TO_R3;
8534 }
8535
8536 ASMSetFlags(pVmxTransient->fEFlags);
8537 VMMRZCallRing3Enable(pVCpu);
8538
8539 return rcStrict;
8540}
8541
8542
8543/**
8544 * Prepares to run guest code in VT-x and we've committed to doing so. This
8545 * means there is no backing out to ring-3 or anywhere else at this
8546 * point.
8547 *
8548 * @param pVM The cross context VM structure.
8549 * @param pVCpu The cross context virtual CPU structure.
8550 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8551 * out-of-sync. Make sure to update the required fields
8552 * before using them.
8553 * @param pVmxTransient Pointer to the VMX transient structure.
8554 *
8555 * @remarks Called with preemption disabled.
8556 * @remarks No-long-jump zone!!!
8557 */
8558static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8559{
8560 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8561 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8562 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8563
8564 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8565 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8566
8567#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8568 if (!CPUMIsGuestFPUStateActive(pVCpu))
8569 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8570 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8571#endif
8572
8573 if ( pVCpu->hm.s.fPreloadGuestFpu
8574 && !CPUMIsGuestFPUStateActive(pVCpu))
8575 {
8576 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8577 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8578 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8579 }
8580
8581 /*
8582 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8583 */
8584 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8585 && pVCpu->hm.s.vmx.cMsrs > 0)
8586 {
8587 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8588 }
8589
8590 /*
8591 * Load the host state bits as we may've been preempted (only happens when
8592 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8593 */
8594 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8595 * any effect to the host state needing to be saved? */
8596 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8597 {
8598 /* This ASSUMES that pfnStartVM has been set up already. */
8599 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8600 AssertRC(rc);
8601 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8602 }
8603 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8604
8605 /*
8606 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8607 */
8608 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8609 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8610 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8611
8612 /* Store status of the shared guest-host state at the time of VM-entry. */
8613#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8614 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8615 {
8616 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8617 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8618 }
8619 else
8620#endif
8621 {
8622 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8623 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8624 }
8625 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8626
8627 /*
8628 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8629 */
8630 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8631 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8632
8633 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8634 RTCPUID idCurrentCpu = pCpu->idCpu;
8635 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8636 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8637 {
8638 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8639 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8640 }
8641
8642 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8643 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8644 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8645 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8646
8647 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8648
8649 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8650 to start executing. */
8651
8652 /*
8653 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8654 */
8655 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8656 {
8657 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8658 {
8659 bool fMsrUpdated;
8660 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8661 AssertRC(rc2);
8662 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8663
8664 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8665 &fMsrUpdated);
8666 AssertRC(rc2);
8667 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8668
8669 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8670 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8671 }
8672 else
8673 {
8674 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8675 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8676 }
8677 }
8678
8679#ifdef VBOX_STRICT
8680 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8681 hmR0VmxCheckHostEferMsr(pVCpu);
8682 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8683#endif
8684#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8685 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8686 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8687 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8688#endif
8689}
8690
8691
8692/**
8693 * Performs some essential restoration of state after running guest code in
8694 * VT-x.
8695 *
8696 * @param pVM The cross context VM structure.
8697 * @param pVCpu The cross context virtual CPU structure.
8698 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8699 * out-of-sync. Make sure to update the required fields
8700 * before using them.
8701 * @param pVmxTransient Pointer to the VMX transient structure.
8702 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8703 *
8704 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8705 *
8706 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8707 * unconditionally when it is safe to do so.
8708 */
8709static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8710{
8711 NOREF(pVM);
8712
8713 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8714
8715 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8716 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8717 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8718 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8719 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8720 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8721
8722 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8723 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8724
8725 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8726 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8727 Assert(!ASMIntAreEnabled());
8728 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8729
8730#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8731 if (CPUMIsGuestFPUStateActive(pVCpu))
8732 {
8733 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8734 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8735 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8736 }
8737#endif
8738
8739#if HC_ARCH_BITS == 64
8740 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8741#endif
8742 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8743#ifdef VBOX_STRICT
8744 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8745#endif
8746 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8747 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8748
8749 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8750 uint32_t uExitReason;
8751 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8752 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8753 AssertRC(rc);
8754 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8755 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8756
8757 /* Update the VM-exit history array. */
8758 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8759
8760 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8761 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8762 {
8763 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8764 pVmxTransient->fVMEntryFailed));
8765 return;
8766 }
8767
8768 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8769 {
8770 /** @todo We can optimize this by only syncing with our force-flags when
8771 * really needed and keeping the VMCS state as it is for most
8772 * VM-exits. */
8773 /* Update the guest interruptibility-state from the VMCS. */
8774 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8775
8776#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8777 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8778 AssertRC(rc);
8779#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8780 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8781 AssertRC(rc);
8782#endif
8783
8784 /*
8785 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8786 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8787 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8788 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8789 */
8790 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8791 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8792 {
8793 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8794 AssertRC(rc);
8795 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8796 }
8797 }
8798}
8799
8800
8801/**
8802 * Runs the guest code using VT-x the normal way.
8803 *
8804 * @returns VBox status code.
8805 * @param pVM The cross context VM structure.
8806 * @param pVCpu The cross context virtual CPU structure.
8807 * @param pCtx Pointer to the guest-CPU context.
8808 *
8809 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8810 */
8811static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8812{
8813 VMXTRANSIENT VmxTransient;
8814 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8815 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8816 uint32_t cLoops = 0;
8817
8818 for (;; cLoops++)
8819 {
8820 Assert(!HMR0SuspendPending());
8821 HMVMX_ASSERT_CPU_SAFE();
8822
8823 /* Preparatory work for running guest code, this may force us to return
8824 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8825 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8826 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8827 if (rcStrict != VINF_SUCCESS)
8828 break;
8829
8830 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8831 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8832 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8833
8834 /* Restore any residual host-state and save any bits shared between host
8835 and guest into the guest-CPU state. Re-enables interrupts! */
8836 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8837
8838 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8839 if (RT_SUCCESS(rcRun))
8840 { /* very likely */ }
8841 else
8842 {
8843 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8844 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8845 return rcRun;
8846 }
8847
8848 /* Profile the VM-exit. */
8849 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8850 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8851 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8852 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8853 HMVMX_START_EXIT_DISPATCH_PROF();
8854
8855 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8856
8857 /* Handle the VM-exit. */
8858#ifdef HMVMX_USE_FUNCTION_TABLE
8859 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8860#else
8861 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8862#endif
8863 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8864 if (rcStrict == VINF_SUCCESS)
8865 {
8866 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8867 continue; /* likely */
8868 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8869 rcStrict = VINF_EM_RAW_INTERRUPT;
8870 }
8871 break;
8872 }
8873
8874 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8875 return rcStrict;
8876}
8877
8878
8879
8880/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8881 * probes.
8882 *
8883 * The following few functions and associated structure contains the bloat
8884 * necessary for providing detailed debug events and dtrace probes as well as
8885 * reliable host side single stepping. This works on the principle of
8886 * "subclassing" the normal execution loop and workers. We replace the loop
8887 * method completely and override selected helpers to add necessary adjustments
8888 * to their core operation.
8889 *
8890 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8891 * any performance for debug and analysis features.
8892 *
8893 * @{
8894 */
8895
8896typedef struct VMXRUNDBGSTATE
8897{
8898 /** The RIP we started executing at. This is for detecting that we stepped. */
8899 uint64_t uRipStart;
8900 /** The CS we started executing with. */
8901 uint16_t uCsStart;
8902
8903 /** Whether we've actually modified the 1st execution control field. */
8904 bool fModifiedProcCtls : 1;
8905 /** Whether we've actually modified the 2nd execution control field. */
8906 bool fModifiedProcCtls2 : 1;
8907 /** Whether we've actually modified the exception bitmap. */
8908 bool fModifiedXcptBitmap : 1;
8909
8910 /** We desire the modified the CR0 mask to be cleared. */
8911 bool fClearCr0Mask : 1;
8912 /** We desire the modified the CR4 mask to be cleared. */
8913 bool fClearCr4Mask : 1;
8914 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8915 uint32_t fCpe1Extra;
8916 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8917 uint32_t fCpe1Unwanted;
8918 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8919 uint32_t fCpe2Extra;
8920 /** Extra stuff we need in */
8921 uint32_t bmXcptExtra;
8922 /** The sequence number of the Dtrace provider settings the state was
8923 * configured against. */
8924 uint32_t uDtraceSettingsSeqNo;
8925 /** Exits to check (one bit per exit). */
8926 uint32_t bmExitsToCheck[3];
8927
8928 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8929 uint32_t fProcCtlsInitial;
8930 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8931 uint32_t fProcCtls2Initial;
8932 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8933 uint32_t bmXcptInitial;
8934} VMXRUNDBGSTATE;
8935AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8936typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8937
8938
8939/**
8940 * Initializes the VMXRUNDBGSTATE structure.
8941 *
8942 * @param pVCpu The cross context virtual CPU structure of the
8943 * calling EMT.
8944 * @param pCtx The CPU register context to go with @a pVCpu.
8945 * @param pDbgState The structure to initialize.
8946 */
8947DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8948{
8949 pDbgState->uRipStart = pCtx->rip;
8950 pDbgState->uCsStart = pCtx->cs.Sel;
8951
8952 pDbgState->fModifiedProcCtls = false;
8953 pDbgState->fModifiedProcCtls2 = false;
8954 pDbgState->fModifiedXcptBitmap = false;
8955 pDbgState->fClearCr0Mask = false;
8956 pDbgState->fClearCr4Mask = false;
8957 pDbgState->fCpe1Extra = 0;
8958 pDbgState->fCpe1Unwanted = 0;
8959 pDbgState->fCpe2Extra = 0;
8960 pDbgState->bmXcptExtra = 0;
8961 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8962 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8963 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8964}
8965
8966
8967/**
8968 * Updates the VMSC fields with changes requested by @a pDbgState.
8969 *
8970 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8971 * immediately before executing guest code, i.e. when interrupts are disabled.
8972 * We don't check status codes here as we cannot easily assert or return in the
8973 * latter case.
8974 *
8975 * @param pVCpu The cross context virtual CPU structure.
8976 * @param pDbgState The debug state.
8977 */
8978DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8979{
8980 /*
8981 * Ensure desired flags in VMCS control fields are set.
8982 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8983 *
8984 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8985 * there should be no stale data in pCtx at this point.
8986 */
8987 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8988 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8989 {
8990 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8991 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8992 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8993 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8994 pDbgState->fModifiedProcCtls = true;
8995 }
8996
8997 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8998 {
8999 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9000 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9001 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9002 pDbgState->fModifiedProcCtls2 = true;
9003 }
9004
9005 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9006 {
9007 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9008 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9009 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9010 pDbgState->fModifiedXcptBitmap = true;
9011 }
9012
9013 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9014 {
9015 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9016 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9017 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9018 }
9019
9020 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9021 {
9022 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9023 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9024 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9025 }
9026}
9027
9028
9029DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9030{
9031 /*
9032 * Restore exit control settings as we may not reenter this function the
9033 * next time around.
9034 */
9035 /* We reload the initial value, trigger what we can of recalculations the
9036 next time around. From the looks of things, that's all that's required atm. */
9037 if (pDbgState->fModifiedProcCtls)
9038 {
9039 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9040 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9041 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9042 AssertRCReturn(rc2, rc2);
9043 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9044 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9045 }
9046
9047 /* We're currently the only ones messing with this one, so just restore the
9048 cached value and reload the field. */
9049 if ( pDbgState->fModifiedProcCtls2
9050 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9051 {
9052 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9053 AssertRCReturn(rc2, rc2);
9054 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9055 }
9056
9057 /* If we've modified the exception bitmap, we restore it and trigger
9058 reloading and partial recalculation the next time around. */
9059 if (pDbgState->fModifiedXcptBitmap)
9060 {
9061 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9062 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9063 }
9064
9065 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9066 if (pDbgState->fClearCr0Mask)
9067 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9068
9069 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9070 if (pDbgState->fClearCr4Mask)
9071 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9072
9073 return rcStrict;
9074}
9075
9076
9077/**
9078 * Configures VM-exit controls for current DBGF and DTrace settings.
9079 *
9080 * This updates @a pDbgState and the VMCS execution control fields to reflect
9081 * the necessary exits demanded by DBGF and DTrace.
9082 *
9083 * @param pVM The cross context VM structure.
9084 * @param pVCpu The cross context virtual CPU structure.
9085 * @param pCtx Pointer to the guest-CPU context.
9086 * @param pDbgState The debug state.
9087 * @param pVmxTransient Pointer to the VMX transient structure. May update
9088 * fUpdateTscOffsettingAndPreemptTimer.
9089 */
9090static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9091 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9092{
9093 /*
9094 * Take down the dtrace serial number so we can spot changes.
9095 */
9096 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9097 ASMCompilerBarrier();
9098
9099 /*
9100 * We'll rebuild most of the middle block of data members (holding the
9101 * current settings) as we go along here, so start by clearing it all.
9102 */
9103 pDbgState->bmXcptExtra = 0;
9104 pDbgState->fCpe1Extra = 0;
9105 pDbgState->fCpe1Unwanted = 0;
9106 pDbgState->fCpe2Extra = 0;
9107 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9108 pDbgState->bmExitsToCheck[i] = 0;
9109
9110 /*
9111 * Software interrupts (INT XXh) - no idea how to trigger these...
9112 */
9113 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9114 || VBOXVMM_INT_SOFTWARE_ENABLED())
9115 {
9116 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9117 }
9118
9119 /*
9120 * Exception bitmap and XCPT events+probes.
9121 */
9122 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9123 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9124 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9125
9126 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9127 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9128 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9129 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9130 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9131 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9132 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9133 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9134 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9135 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9136 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9137 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9138 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9139 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9140 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9141 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9142 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9143 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9144
9145 if (pDbgState->bmXcptExtra)
9146 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9147
9148 /*
9149 * Process events and probes for VM exits, making sure we get the wanted exits.
9150 *
9151 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9152 * So, when adding/changing/removing please don't forget to update it.
9153 *
9154 * Some of the macros are picking up local variables to save horizontal space,
9155 * (being able to see it in a table is the lesser evil here).
9156 */
9157#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9158 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9159 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9160#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9161 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9162 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9163 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9164 } else do { } while (0)
9165#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9166 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9167 { \
9168 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9169 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9170 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9171 } else do { } while (0)
9172#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9173 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9174 { \
9175 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9176 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9177 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9178 } else do { } while (0)
9179#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9180 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9181 { \
9182 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9183 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9184 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9185 } else do { } while (0)
9186
9187 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9188 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9189 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9190 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9191 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9192
9193 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9194 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9195 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9196 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9197 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9198 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9199 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9200 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9201 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9202 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9203 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9204 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9205 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9206 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9207 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9208 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9209 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9210 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9211 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9212 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9213 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9214 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9215 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9216 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9217 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9218 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9219 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9220 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9221 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9222 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9223 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9224 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9225 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9226 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9227 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9228 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9229
9230 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9231 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9232 {
9233 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9234 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9235 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9236 AssertRC(rc2);
9237
9238#if 0 /** @todo fix me */
9239 pDbgState->fClearCr0Mask = true;
9240 pDbgState->fClearCr4Mask = true;
9241#endif
9242 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9243 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9244 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9245 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9246 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9247 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9248 require clearing here and in the loop if we start using it. */
9249 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9250 }
9251 else
9252 {
9253 if (pDbgState->fClearCr0Mask)
9254 {
9255 pDbgState->fClearCr0Mask = false;
9256 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9257 }
9258 if (pDbgState->fClearCr4Mask)
9259 {
9260 pDbgState->fClearCr4Mask = false;
9261 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9262 }
9263 }
9264 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9265 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9266
9267 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9268 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9269 {
9270 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9271 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9272 }
9273 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9274 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9275
9276 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9277 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9278 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9279 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9280 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9281 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9282 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9283 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9284#if 0 /** @todo too slow, fix handler. */
9285 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9286#endif
9287 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9288
9289 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9290 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9291 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9292 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9293 {
9294 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9295 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9296 }
9297 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9298 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9299 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9300 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9301
9302 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9303 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9304 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9305 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9306 {
9307 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9308 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9309 }
9310 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9311 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9312 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9313 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9314
9315 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9316 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9317 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9318 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9319 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9320 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9321 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9322 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9323 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9324 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9325 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9326 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9327 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9328 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9329 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9330 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9331 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9332 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9333 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9334 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9335 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9336 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9337
9338#undef IS_EITHER_ENABLED
9339#undef SET_ONLY_XBM_IF_EITHER_EN
9340#undef SET_CPE1_XBM_IF_EITHER_EN
9341#undef SET_CPEU_XBM_IF_EITHER_EN
9342#undef SET_CPE2_XBM_IF_EITHER_EN
9343
9344 /*
9345 * Sanitize the control stuff.
9346 */
9347 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9348 if (pDbgState->fCpe2Extra)
9349 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9350 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9351 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9352 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9353 {
9354 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9355 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9356 }
9357
9358 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9359 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9360 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9361 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9362}
9363
9364
9365/**
9366 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9367 *
9368 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9369 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9370 * either.
9371 *
9372 * @returns Strict VBox status code (i.e. informational status codes too).
9373 * @param pVM The cross context VM structure.
9374 * @param pVCpu The cross context virtual CPU structure.
9375 * @param pMixedCtx Pointer to the guest-CPU context.
9376 * @param pVmxTransient Pointer to the VMX-transient structure.
9377 * @param uExitReason The VM-exit reason.
9378 *
9379 * @remarks The name of this function is displayed by dtrace, so keep it short
9380 * and to the point. No longer than 33 chars long, please.
9381 */
9382static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9383 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9384{
9385 /*
9386 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9387 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9388 *
9389 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9390 * does. Must add/change/remove both places. Same ordering, please.
9391 *
9392 * Added/removed events must also be reflected in the next section
9393 * where we dispatch dtrace events.
9394 */
9395 bool fDtrace1 = false;
9396 bool fDtrace2 = false;
9397 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9398 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9399 uint32_t uEventArg = 0;
9400#define SET_EXIT(a_EventSubName) \
9401 do { \
9402 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9403 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9404 } while (0)
9405#define SET_BOTH(a_EventSubName) \
9406 do { \
9407 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9408 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9409 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9410 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9411 } while (0)
9412 switch (uExitReason)
9413 {
9414 case VMX_EXIT_MTF:
9415 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9416
9417 case VMX_EXIT_XCPT_OR_NMI:
9418 {
9419 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9420 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9421 {
9422 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9423 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9424 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9425 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9426 {
9427 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9428 {
9429 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9430 uEventArg = pVmxTransient->uExitIntErrorCode;
9431 }
9432 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9433 switch (enmEvent1)
9434 {
9435 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9436 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9437 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9438 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9439 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9440 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9441 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9442 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9443 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9444 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9445 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9446 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9447 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9448 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9449 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9450 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9451 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9452 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9453 default: break;
9454 }
9455 }
9456 else
9457 AssertFailed();
9458 break;
9459
9460 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9461 uEventArg = idxVector;
9462 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9463 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9464 break;
9465 }
9466 break;
9467 }
9468
9469 case VMX_EXIT_TRIPLE_FAULT:
9470 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9471 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9472 break;
9473 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9474 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9475 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9476 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9477 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9478
9479 /* Instruction specific VM-exits: */
9480 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9481 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9482 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9483 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9484 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9485 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9486 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9487 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9488 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9489 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9490 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9491 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9492 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9493 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9494 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9495 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9496 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9497 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9498 case VMX_EXIT_MOV_CRX:
9499 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9500/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9501* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9502 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9503 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9504 SET_BOTH(CRX_READ);
9505 else
9506 SET_BOTH(CRX_WRITE);
9507 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9508 break;
9509 case VMX_EXIT_MOV_DRX:
9510 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9511 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9512 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9513 SET_BOTH(DRX_READ);
9514 else
9515 SET_BOTH(DRX_WRITE);
9516 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9517 break;
9518 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9519 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9520 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9521 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9522 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9523 case VMX_EXIT_XDTR_ACCESS:
9524 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9525 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9526 {
9527 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9528 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9529 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9530 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9531 }
9532 break;
9533
9534 case VMX_EXIT_TR_ACCESS:
9535 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9536 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9537 {
9538 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9539 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9540 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9541 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9542 }
9543 break;
9544
9545 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9546 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9547 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9548 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9549 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9550 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9551 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9552 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9553 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9554 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9555 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9556
9557 /* Events that aren't relevant at this point. */
9558 case VMX_EXIT_EXT_INT:
9559 case VMX_EXIT_INT_WINDOW:
9560 case VMX_EXIT_NMI_WINDOW:
9561 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9562 case VMX_EXIT_PREEMPT_TIMER:
9563 case VMX_EXIT_IO_INSTR:
9564 break;
9565
9566 /* Errors and unexpected events. */
9567 case VMX_EXIT_INIT_SIGNAL:
9568 case VMX_EXIT_SIPI:
9569 case VMX_EXIT_IO_SMI:
9570 case VMX_EXIT_SMI:
9571 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9572 case VMX_EXIT_ERR_MSR_LOAD:
9573 case VMX_EXIT_ERR_MACHINE_CHECK:
9574 break;
9575
9576 default:
9577 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9578 break;
9579 }
9580#undef SET_BOTH
9581#undef SET_EXIT
9582
9583 /*
9584 * Dtrace tracepoints go first. We do them here at once so we don't
9585 * have to copy the guest state saving and stuff a few dozen times.
9586 * Down side is that we've got to repeat the switch, though this time
9587 * we use enmEvent since the probes are a subset of what DBGF does.
9588 */
9589 if (fDtrace1 || fDtrace2)
9590 {
9591 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9592 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9593 switch (enmEvent1)
9594 {
9595 /** @todo consider which extra parameters would be helpful for each probe. */
9596 case DBGFEVENT_END: break;
9597 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9598 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9599 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9600 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9601 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9602 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9603 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9604 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9605 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9606 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9607 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9608 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9609 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9610 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9611 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9612 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9613 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9614 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9615 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9616 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9617 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9618 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9619 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9620 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9621 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9622 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9623 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9624 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9625 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9626 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9627 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9628 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9629 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9630 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9631 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9632 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9633 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9634 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9635 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9636 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9637 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9638 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9639 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9640 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9641 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9642 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9643 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9644 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9645 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9646 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9647 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9648 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9649 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9650 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9651 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9652 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9653 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9654 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9655 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9656 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9657 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9658 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9659 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9660 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9661 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9662 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9663 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9664 }
9665 switch (enmEvent2)
9666 {
9667 /** @todo consider which extra parameters would be helpful for each probe. */
9668 case DBGFEVENT_END: break;
9669 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9670 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9671 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9672 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9673 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9674 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9675 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9676 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9677 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9678 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9679 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9680 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9681 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9682 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9683 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9684 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9685 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9686 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9687 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9688 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9689 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9690 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9691 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9692 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9693 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9694 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9695 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9696 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9697 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9698 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9699 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9700 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9701 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9702 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9703 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9704 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9705 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9706 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9707 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9708 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9709 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9710 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9711 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9712 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9713 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9714 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9715 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9716 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9717 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9718 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9719 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9720 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9721 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9722 }
9723 }
9724
9725 /*
9726 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9727 * the DBGF call will do a full check).
9728 *
9729 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9730 * Note! If we have to events, we prioritize the first, i.e. the instruction
9731 * one, in order to avoid event nesting.
9732 */
9733 if ( enmEvent1 != DBGFEVENT_END
9734 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9735 {
9736 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9737 if (rcStrict != VINF_SUCCESS)
9738 return rcStrict;
9739 }
9740 else if ( enmEvent2 != DBGFEVENT_END
9741 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9742 {
9743 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9744 if (rcStrict != VINF_SUCCESS)
9745 return rcStrict;
9746 }
9747
9748 return VINF_SUCCESS;
9749}
9750
9751
9752/**
9753 * Single-stepping VM-exit filtering.
9754 *
9755 * This is preprocessing the exits and deciding whether we've gotten far enough
9756 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9757 * performed.
9758 *
9759 * @returns Strict VBox status code (i.e. informational status codes too).
9760 * @param pVM The cross context VM structure.
9761 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9762 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9763 * out-of-sync. Make sure to update the required
9764 * fields before using them.
9765 * @param pVmxTransient Pointer to the VMX-transient structure.
9766 * @param uExitReason The VM-exit reason.
9767 * @param pDbgState The debug state.
9768 */
9769DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9770 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9771{
9772 /*
9773 * Expensive (saves context) generic dtrace exit probe.
9774 */
9775 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9776 { /* more likely */ }
9777 else
9778 {
9779 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9780 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9781 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9782 }
9783
9784 /*
9785 * Check for host NMI, just to get that out of the way.
9786 */
9787 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9788 { /* normally likely */ }
9789 else
9790 {
9791 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9792 AssertRCReturn(rc2, rc2);
9793 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9794 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9795 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9796 }
9797
9798 /*
9799 * Check for single stepping event if we're stepping.
9800 */
9801 if (pVCpu->hm.s.fSingleInstruction)
9802 {
9803 switch (uExitReason)
9804 {
9805 case VMX_EXIT_MTF:
9806 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9807
9808 /* Various events: */
9809 case VMX_EXIT_XCPT_OR_NMI:
9810 case VMX_EXIT_EXT_INT:
9811 case VMX_EXIT_TRIPLE_FAULT:
9812 case VMX_EXIT_INT_WINDOW:
9813 case VMX_EXIT_NMI_WINDOW:
9814 case VMX_EXIT_TASK_SWITCH:
9815 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9816 case VMX_EXIT_APIC_ACCESS:
9817 case VMX_EXIT_EPT_VIOLATION:
9818 case VMX_EXIT_EPT_MISCONFIG:
9819 case VMX_EXIT_PREEMPT_TIMER:
9820
9821 /* Instruction specific VM-exits: */
9822 case VMX_EXIT_CPUID:
9823 case VMX_EXIT_GETSEC:
9824 case VMX_EXIT_HLT:
9825 case VMX_EXIT_INVD:
9826 case VMX_EXIT_INVLPG:
9827 case VMX_EXIT_RDPMC:
9828 case VMX_EXIT_RDTSC:
9829 case VMX_EXIT_RSM:
9830 case VMX_EXIT_VMCALL:
9831 case VMX_EXIT_VMCLEAR:
9832 case VMX_EXIT_VMLAUNCH:
9833 case VMX_EXIT_VMPTRLD:
9834 case VMX_EXIT_VMPTRST:
9835 case VMX_EXIT_VMREAD:
9836 case VMX_EXIT_VMRESUME:
9837 case VMX_EXIT_VMWRITE:
9838 case VMX_EXIT_VMXOFF:
9839 case VMX_EXIT_VMXON:
9840 case VMX_EXIT_MOV_CRX:
9841 case VMX_EXIT_MOV_DRX:
9842 case VMX_EXIT_IO_INSTR:
9843 case VMX_EXIT_RDMSR:
9844 case VMX_EXIT_WRMSR:
9845 case VMX_EXIT_MWAIT:
9846 case VMX_EXIT_MONITOR:
9847 case VMX_EXIT_PAUSE:
9848 case VMX_EXIT_XDTR_ACCESS:
9849 case VMX_EXIT_TR_ACCESS:
9850 case VMX_EXIT_INVEPT:
9851 case VMX_EXIT_RDTSCP:
9852 case VMX_EXIT_INVVPID:
9853 case VMX_EXIT_WBINVD:
9854 case VMX_EXIT_XSETBV:
9855 case VMX_EXIT_RDRAND:
9856 case VMX_EXIT_INVPCID:
9857 case VMX_EXIT_VMFUNC:
9858 case VMX_EXIT_RDSEED:
9859 case VMX_EXIT_XSAVES:
9860 case VMX_EXIT_XRSTORS:
9861 {
9862 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9863 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9864 AssertRCReturn(rc2, rc2);
9865 if ( pMixedCtx->rip != pDbgState->uRipStart
9866 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9867 return VINF_EM_DBG_STEPPED;
9868 break;
9869 }
9870
9871 /* Errors and unexpected events: */
9872 case VMX_EXIT_INIT_SIGNAL:
9873 case VMX_EXIT_SIPI:
9874 case VMX_EXIT_IO_SMI:
9875 case VMX_EXIT_SMI:
9876 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9877 case VMX_EXIT_ERR_MSR_LOAD:
9878 case VMX_EXIT_ERR_MACHINE_CHECK:
9879 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9880 break;
9881
9882 default:
9883 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9884 break;
9885 }
9886 }
9887
9888 /*
9889 * Check for debugger event breakpoints and dtrace probes.
9890 */
9891 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9892 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9893 {
9894 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9895 if (rcStrict != VINF_SUCCESS)
9896 return rcStrict;
9897 }
9898
9899 /*
9900 * Normal processing.
9901 */
9902#ifdef HMVMX_USE_FUNCTION_TABLE
9903 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9904#else
9905 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9906#endif
9907}
9908
9909
9910/**
9911 * Single steps guest code using VT-x.
9912 *
9913 * @returns Strict VBox status code (i.e. informational status codes too).
9914 * @param pVM The cross context VM structure.
9915 * @param pVCpu The cross context virtual CPU structure.
9916 * @param pCtx Pointer to the guest-CPU context.
9917 *
9918 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9919 */
9920static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9921{
9922 VMXTRANSIENT VmxTransient;
9923 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9924
9925 /* Set HMCPU indicators. */
9926 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9927 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9928 pVCpu->hm.s.fDebugWantRdTscExit = false;
9929 pVCpu->hm.s.fUsingDebugLoop = true;
9930
9931 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9932 VMXRUNDBGSTATE DbgState;
9933 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9934 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
9935
9936 /*
9937 * The loop.
9938 */
9939 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9940 for (uint32_t cLoops = 0; ; cLoops++)
9941 {
9942 Assert(!HMR0SuspendPending());
9943 HMVMX_ASSERT_CPU_SAFE();
9944 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9945
9946 /*
9947 * Preparatory work for running guest code, this may force us to return
9948 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9949 */
9950 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9951 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9952 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
9953 if (rcStrict != VINF_SUCCESS)
9954 break;
9955
9956 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9957 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9958
9959 /*
9960 * Now we can run the guest code.
9961 */
9962 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9963
9964 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9965
9966 /*
9967 * Restore any residual host-state and save any bits shared between host
9968 * and guest into the guest-CPU state. Re-enables interrupts!
9969 */
9970 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
9971
9972 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9973 if (RT_SUCCESS(rcRun))
9974 { /* very likely */ }
9975 else
9976 {
9977 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9978 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9979 return rcRun;
9980 }
9981
9982 /* Profile the VM-exit. */
9983 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9984 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9985 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9986 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9987 HMVMX_START_EXIT_DISPATCH_PROF();
9988
9989 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9990
9991 /*
9992 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9993 */
9994 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9995 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9996 if (rcStrict != VINF_SUCCESS)
9997 break;
9998 if (cLoops > pVM->hm.s.cMaxResumeLoops)
9999 {
10000 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10001 rcStrict = VINF_EM_RAW_INTERRUPT;
10002 break;
10003 }
10004
10005 /*
10006 * Stepping: Did the RIP change, if so, consider it a single step.
10007 * Otherwise, make sure one of the TFs gets set.
10008 */
10009 if (fStepping)
10010 {
10011 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10012 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10013 AssertRCReturn(rc2, rc2);
10014 if ( pCtx->rip != DbgState.uRipStart
10015 || pCtx->cs.Sel != DbgState.uCsStart)
10016 {
10017 rcStrict = VINF_EM_DBG_STEPPED;
10018 break;
10019 }
10020 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10021 }
10022
10023 /*
10024 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10025 */
10026 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10027 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10028 }
10029
10030 /*
10031 * Clear the X86_EFL_TF if necessary.
10032 */
10033 if (pVCpu->hm.s.fClearTrapFlag)
10034 {
10035 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10036 AssertRCReturn(rc2, rc2);
10037 pVCpu->hm.s.fClearTrapFlag = false;
10038 pCtx->eflags.Bits.u1TF = 0;
10039 }
10040 /** @todo there seems to be issues with the resume flag when the monitor trap
10041 * flag is pending without being used. Seen early in bios init when
10042 * accessing APIC page in protected mode. */
10043
10044 /*
10045 * Restore VM-exit control settings as we may not reenter this function the
10046 * next time around.
10047 */
10048 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10049
10050 /* Restore HMCPU indicators. */
10051 pVCpu->hm.s.fUsingDebugLoop = false;
10052 pVCpu->hm.s.fDebugWantRdTscExit = false;
10053 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10054
10055 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10056 return rcStrict;
10057}
10058
10059
10060/** @} */
10061
10062
10063/**
10064 * Checks if any expensive dtrace probes are enabled and we should go to the
10065 * debug loop.
10066 *
10067 * @returns true if we should use debug loop, false if not.
10068 */
10069static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10070{
10071 /* It's probably faster to OR the raw 32-bit counter variables together.
10072 Since the variables are in an array and the probes are next to one
10073 another (more or less), we have good locality. So, better read
10074 eight-nine cache lines ever time and only have one conditional, than
10075 128+ conditionals, right? */
10076 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10077 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10078 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10079 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10080 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10081 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10082 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10083 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10084 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10085 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10086 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10087 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10088 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10089 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10090 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10091 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10092 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10093 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10094 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10095 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10096 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10097 ) != 0
10098 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10099 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10100 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10101 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10102 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10103 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10104 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10105 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10106 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10107 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10108 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10109 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10110 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10111 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10112 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10113 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10114 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10115 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10116 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10117 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10118 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10119 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10120 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10121 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10122 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10123 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10124 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10125 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10126 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10127 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10128 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10129 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10130 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10131 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10132 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10133 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10134 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10135 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10136 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10137 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10138 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10139 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10140 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10141 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10142 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10143 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10144 ) != 0
10145 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10146 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10147 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10148 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10149 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10150 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10151 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10152 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10153 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10154 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10155 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10156 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10157 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10158 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10159 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10160 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10161 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10162 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10163 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10164 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10165 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10166 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10167 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10168 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10169 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10170 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10171 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10172 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10173 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10174 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10175 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10176 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10177 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10178 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10179 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10180 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10181 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10182 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10183 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10184 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10185 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10186 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10187 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10188 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10189 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10190 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10191 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10192 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10193 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10194 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10195 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10196 ) != 0;
10197}
10198
10199
10200/**
10201 * Runs the guest code using VT-x.
10202 *
10203 * @returns Strict VBox status code (i.e. informational status codes too).
10204 * @param pVM The cross context VM structure.
10205 * @param pVCpu The cross context virtual CPU structure.
10206 * @param pCtx Pointer to the guest-CPU context.
10207 */
10208VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10209{
10210 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10211 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10212 HMVMX_ASSERT_PREEMPT_SAFE();
10213
10214 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10215
10216 VBOXSTRICTRC rcStrict;
10217 if ( !pVCpu->hm.s.fUseDebugLoop
10218 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10219 && !DBGFIsStepping(pVCpu) )
10220 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10221 else
10222 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10223
10224 if (rcStrict == VERR_EM_INTERPRETER)
10225 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10226 else if (rcStrict == VINF_EM_RESET)
10227 rcStrict = VINF_EM_TRIPLE_FAULT;
10228
10229 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10230 if (RT_FAILURE(rc2))
10231 {
10232 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10233 rcStrict = rc2;
10234 }
10235 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10236 return rcStrict;
10237}
10238
10239
10240#ifndef HMVMX_USE_FUNCTION_TABLE
10241DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10242{
10243# ifdef DEBUG_ramshankar
10244# define RETURN_EXIT_CALL(a_CallExpr) \
10245 do { \
10246 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10247 VBOXSTRICTRC rcStrict = a_CallExpr; \
10248 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10249 return rcStrict; \
10250 } while (0)
10251# else
10252# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10253# endif
10254 switch (rcReason)
10255 {
10256 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10257 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10258 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10259 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10260 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10261 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10262 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10263 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10264 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10265 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10266 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10267 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10268 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10269 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10270 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10271 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10272 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10273 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10274 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10275 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10276 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10277 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10278 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10279 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10280 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10281 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10282 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10283 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10284 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10285 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10286 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10287 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10288 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10289 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10290
10291 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10292 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10293 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10294 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10295 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10296 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10297 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10298 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10299 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10300
10301 case VMX_EXIT_VMCLEAR:
10302 case VMX_EXIT_VMLAUNCH:
10303 case VMX_EXIT_VMPTRLD:
10304 case VMX_EXIT_VMPTRST:
10305 case VMX_EXIT_VMREAD:
10306 case VMX_EXIT_VMRESUME:
10307 case VMX_EXIT_VMWRITE:
10308 case VMX_EXIT_VMXOFF:
10309 case VMX_EXIT_VMXON:
10310 case VMX_EXIT_INVEPT:
10311 case VMX_EXIT_INVVPID:
10312 case VMX_EXIT_VMFUNC:
10313 case VMX_EXIT_XSAVES:
10314 case VMX_EXIT_XRSTORS:
10315 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10316 case VMX_EXIT_RESERVED_60:
10317 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10318 case VMX_EXIT_RESERVED_62:
10319 default:
10320 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10321 }
10322#undef RETURN_EXIT_CALL
10323}
10324#endif /* !HMVMX_USE_FUNCTION_TABLE */
10325
10326
10327#ifdef VBOX_STRICT
10328/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10329# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10330 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10331
10332# define HMVMX_ASSERT_PREEMPT_CPUID() \
10333 do { \
10334 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10335 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10336 } while (0)
10337
10338# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10339 do { \
10340 AssertPtr(pVCpu); \
10341 AssertPtr(pMixedCtx); \
10342 AssertPtr(pVmxTransient); \
10343 Assert(pVmxTransient->fVMEntryFailed == false); \
10344 Assert(ASMIntAreEnabled()); \
10345 HMVMX_ASSERT_PREEMPT_SAFE(); \
10346 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10347 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10348 HMVMX_ASSERT_PREEMPT_SAFE(); \
10349 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10350 HMVMX_ASSERT_PREEMPT_CPUID(); \
10351 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10352 } while (0)
10353
10354# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10355 do { \
10356 Log4Func(("\n")); \
10357 } while (0)
10358#else /* nonstrict builds: */
10359# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10360 do { \
10361 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10362 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10363 } while (0)
10364# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10365#endif
10366
10367
10368/**
10369 * Advances the guest RIP after reading it from the VMCS.
10370 *
10371 * @returns VBox status code, no informational status codes.
10372 * @param pVCpu The cross context virtual CPU structure.
10373 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10374 * out-of-sync. Make sure to update the required fields
10375 * before using them.
10376 * @param pVmxTransient Pointer to the VMX transient structure.
10377 *
10378 * @remarks No-long-jump zone!!!
10379 */
10380DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10381{
10382 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10383 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10384 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10385 AssertRCReturn(rc, rc);
10386
10387 pMixedCtx->rip += pVmxTransient->cbInstr;
10388 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10389
10390 /*
10391 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10392 * pending debug exception field as it takes care of priority of events.
10393 *
10394 * See Intel spec. 32.2.1 "Debug Exceptions".
10395 */
10396 if ( !pVCpu->hm.s.fSingleInstruction
10397 && pMixedCtx->eflags.Bits.u1TF)
10398 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10399
10400 return VINF_SUCCESS;
10401}
10402
10403
10404/**
10405 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10406 * and update error record fields accordingly.
10407 *
10408 * @return VMX_IGS_* return codes.
10409 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10410 * wrong with the guest state.
10411 *
10412 * @param pVM The cross context VM structure.
10413 * @param pVCpu The cross context virtual CPU structure.
10414 * @param pCtx Pointer to the guest-CPU state.
10415 *
10416 * @remarks This function assumes our cache of the VMCS controls
10417 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10418 */
10419static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10420{
10421#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10422#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10423 uError = (err); \
10424 break; \
10425 } else do { } while (0)
10426
10427 int rc;
10428 uint32_t uError = VMX_IGS_ERROR;
10429 uint32_t u32Val;
10430 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10431
10432 do
10433 {
10434 /*
10435 * CR0.
10436 */
10437 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10438 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10439 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10440 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10441 if (fUnrestrictedGuest)
10442 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10443
10444 uint32_t u32GuestCR0;
10445 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10446 AssertRCBreak(rc);
10447 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10448 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10449 if ( !fUnrestrictedGuest
10450 && (u32GuestCR0 & X86_CR0_PG)
10451 && !(u32GuestCR0 & X86_CR0_PE))
10452 {
10453 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10454 }
10455
10456 /*
10457 * CR4.
10458 */
10459 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10460 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10461
10462 uint32_t u32GuestCR4;
10463 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10464 AssertRCBreak(rc);
10465 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10466 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10467
10468 /*
10469 * IA32_DEBUGCTL MSR.
10470 */
10471 uint64_t u64Val;
10472 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10473 AssertRCBreak(rc);
10474 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10475 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10476 {
10477 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10478 }
10479 uint64_t u64DebugCtlMsr = u64Val;
10480
10481#ifdef VBOX_STRICT
10482 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10483 AssertRCBreak(rc);
10484 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10485#endif
10486 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10487
10488 /*
10489 * RIP and RFLAGS.
10490 */
10491 uint32_t u32Eflags;
10492#if HC_ARCH_BITS == 64
10493 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10494 AssertRCBreak(rc);
10495 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10496 if ( !fLongModeGuest
10497 || !pCtx->cs.Attr.n.u1Long)
10498 {
10499 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10500 }
10501 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10502 * must be identical if the "IA-32e mode guest" VM-entry
10503 * control is 1 and CS.L is 1. No check applies if the
10504 * CPU supports 64 linear-address bits. */
10505
10506 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10507 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10508 AssertRCBreak(rc);
10509 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10510 VMX_IGS_RFLAGS_RESERVED);
10511 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10512 u32Eflags = u64Val;
10513#else
10514 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10515 AssertRCBreak(rc);
10516 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10517 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10518#endif
10519
10520 if ( fLongModeGuest
10521 || ( fUnrestrictedGuest
10522 && !(u32GuestCR0 & X86_CR0_PE)))
10523 {
10524 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10525 }
10526
10527 uint32_t u32EntryInfo;
10528 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10529 AssertRCBreak(rc);
10530 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10531 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10532 {
10533 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10534 }
10535
10536 /*
10537 * 64-bit checks.
10538 */
10539#if HC_ARCH_BITS == 64
10540 if (fLongModeGuest)
10541 {
10542 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10543 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10544 }
10545
10546 if ( !fLongModeGuest
10547 && (u32GuestCR4 & X86_CR4_PCIDE))
10548 {
10549 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10550 }
10551
10552 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10553 * 51:32 beyond the processor's physical-address width are 0. */
10554
10555 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10556 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10557 {
10558 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10559 }
10560
10561 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10562 AssertRCBreak(rc);
10563 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10564
10565 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10566 AssertRCBreak(rc);
10567 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10568#endif
10569
10570 /*
10571 * PERF_GLOBAL MSR.
10572 */
10573 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10574 {
10575 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10576 AssertRCBreak(rc);
10577 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10578 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10579 }
10580
10581 /*
10582 * PAT MSR.
10583 */
10584 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10585 {
10586 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10587 AssertRCBreak(rc);
10588 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10589 for (unsigned i = 0; i < 8; i++)
10590 {
10591 uint8_t u8Val = (u64Val & 0xff);
10592 if ( u8Val != 0 /* UC */
10593 && u8Val != 1 /* WC */
10594 && u8Val != 4 /* WT */
10595 && u8Val != 5 /* WP */
10596 && u8Val != 6 /* WB */
10597 && u8Val != 7 /* UC- */)
10598 {
10599 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10600 }
10601 u64Val >>= 8;
10602 }
10603 }
10604
10605 /*
10606 * EFER MSR.
10607 */
10608 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10609 {
10610 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10611 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10612 AssertRCBreak(rc);
10613 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10614 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10615 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10616 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10617 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10618 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10619 || !(u32GuestCR0 & X86_CR0_PG)
10620 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10621 VMX_IGS_EFER_LMA_LME_MISMATCH);
10622 }
10623
10624 /*
10625 * Segment registers.
10626 */
10627 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10628 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10629 if (!(u32Eflags & X86_EFL_VM))
10630 {
10631 /* CS */
10632 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10633 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10634 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10635 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10636 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10637 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10638 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10639 /* CS cannot be loaded with NULL in protected mode. */
10640 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10641 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10642 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10643 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10644 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10645 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10646 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10647 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10648 else
10649 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10650
10651 /* SS */
10652 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10653 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10654 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10655 if ( !(pCtx->cr0 & X86_CR0_PE)
10656 || pCtx->cs.Attr.n.u4Type == 3)
10657 {
10658 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10659 }
10660 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10661 {
10662 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10663 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10664 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10665 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10666 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10667 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10668 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10669 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10670 }
10671
10672 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10673 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10674 {
10675 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10676 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10677 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10678 || pCtx->ds.Attr.n.u4Type > 11
10679 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10680 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10681 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10682 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10683 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10684 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10685 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10686 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10687 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10688 }
10689 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10690 {
10691 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10692 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10693 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10694 || pCtx->es.Attr.n.u4Type > 11
10695 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10696 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10697 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10698 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10699 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10700 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10701 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10702 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10703 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10704 }
10705 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10706 {
10707 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10708 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10709 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10710 || pCtx->fs.Attr.n.u4Type > 11
10711 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10712 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10713 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10714 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10715 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10716 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10717 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10718 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10719 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10720 }
10721 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10722 {
10723 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10724 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10725 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10726 || pCtx->gs.Attr.n.u4Type > 11
10727 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10728 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10729 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10730 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10731 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10732 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10733 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10734 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10735 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10736 }
10737 /* 64-bit capable CPUs. */
10738#if HC_ARCH_BITS == 64
10739 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10740 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10741 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10742 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10743 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10744 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10745 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10746 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10747 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10748 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10749 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10750#endif
10751 }
10752 else
10753 {
10754 /* V86 mode checks. */
10755 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10756 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10757 {
10758 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10759 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10760 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10761 }
10762 else
10763 {
10764 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10765 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10766 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10767 }
10768
10769 /* CS */
10770 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10771 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10772 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10773 /* SS */
10774 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10775 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10776 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10777 /* DS */
10778 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10779 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10780 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10781 /* ES */
10782 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10783 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10784 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10785 /* FS */
10786 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10787 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10788 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10789 /* GS */
10790 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10791 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10792 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10793 /* 64-bit capable CPUs. */
10794#if HC_ARCH_BITS == 64
10795 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10796 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10797 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10798 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10799 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10800 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10801 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10802 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10803 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10804 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10805 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10806#endif
10807 }
10808
10809 /*
10810 * TR.
10811 */
10812 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10813 /* 64-bit capable CPUs. */
10814#if HC_ARCH_BITS == 64
10815 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10816#endif
10817 if (fLongModeGuest)
10818 {
10819 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10820 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10821 }
10822 else
10823 {
10824 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10825 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10826 VMX_IGS_TR_ATTR_TYPE_INVALID);
10827 }
10828 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10829 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10830 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10831 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10832 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10833 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10834 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10835 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10836
10837 /*
10838 * GDTR and IDTR.
10839 */
10840#if HC_ARCH_BITS == 64
10841 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10842 AssertRCBreak(rc);
10843 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10844
10845 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10846 AssertRCBreak(rc);
10847 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10848#endif
10849
10850 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10851 AssertRCBreak(rc);
10852 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10853
10854 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10855 AssertRCBreak(rc);
10856 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10857
10858 /*
10859 * Guest Non-Register State.
10860 */
10861 /* Activity State. */
10862 uint32_t u32ActivityState;
10863 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10864 AssertRCBreak(rc);
10865 HMVMX_CHECK_BREAK( !u32ActivityState
10866 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10867 VMX_IGS_ACTIVITY_STATE_INVALID);
10868 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10869 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10870 uint32_t u32IntrState;
10871 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10872 AssertRCBreak(rc);
10873 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10874 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10875 {
10876 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10877 }
10878
10879 /** @todo Activity state and injecting interrupts. Left as a todo since we
10880 * currently don't use activity states but ACTIVE. */
10881
10882 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10883 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10884
10885 /* Guest interruptibility-state. */
10886 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10887 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10888 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10889 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10890 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10891 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10892 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10893 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10894 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10895 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10896 {
10897 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10898 {
10899 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10900 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10901 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10902 }
10903 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10904 {
10905 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10906 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10907 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10908 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10909 }
10910 }
10911 /** @todo Assumes the processor is not in SMM. */
10912 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10913 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10914 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10915 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10916 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10917 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10918 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10919 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10920 {
10921 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10922 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10923 }
10924
10925 /* Pending debug exceptions. */
10926#if HC_ARCH_BITS == 64
10927 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10928 AssertRCBreak(rc);
10929 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10930 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10931 u32Val = u64Val; /* For pending debug exceptions checks below. */
10932#else
10933 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10934 AssertRCBreak(rc);
10935 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10936 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10937#endif
10938
10939 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10940 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10941 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10942 {
10943 if ( (u32Eflags & X86_EFL_TF)
10944 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10945 {
10946 /* Bit 14 is PendingDebug.BS. */
10947 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10948 }
10949 if ( !(u32Eflags & X86_EFL_TF)
10950 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10951 {
10952 /* Bit 14 is PendingDebug.BS. */
10953 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10954 }
10955 }
10956
10957 /* VMCS link pointer. */
10958 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10959 AssertRCBreak(rc);
10960 if (u64Val != UINT64_C(0xffffffffffffffff))
10961 {
10962 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10963 /** @todo Bits beyond the processor's physical-address width MBZ. */
10964 /** @todo 32-bit located in memory referenced by value of this field (as a
10965 * physical address) must contain the processor's VMCS revision ID. */
10966 /** @todo SMM checks. */
10967 }
10968
10969 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10970 * not using Nested Paging? */
10971 if ( pVM->hm.s.fNestedPaging
10972 && !fLongModeGuest
10973 && CPUMIsGuestInPAEModeEx(pCtx))
10974 {
10975 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10976 AssertRCBreak(rc);
10977 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10978
10979 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10980 AssertRCBreak(rc);
10981 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10982
10983 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10984 AssertRCBreak(rc);
10985 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10986
10987 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10988 AssertRCBreak(rc);
10989 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10990 }
10991
10992 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10993 if (uError == VMX_IGS_ERROR)
10994 uError = VMX_IGS_REASON_NOT_FOUND;
10995 } while (0);
10996
10997 pVCpu->hm.s.u32HMError = uError;
10998 return uError;
10999
11000#undef HMVMX_ERROR_BREAK
11001#undef HMVMX_CHECK_BREAK
11002}
11003
11004/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11005/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11006/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11007
11008/** @name VM-exit handlers.
11009 * @{
11010 */
11011
11012/**
11013 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11014 */
11015HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11016{
11017 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11018 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11019 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11020 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11021 return VINF_SUCCESS;
11022 return VINF_EM_RAW_INTERRUPT;
11023}
11024
11025
11026/**
11027 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11028 */
11029HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11030{
11031 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11032 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11033
11034 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11035 AssertRCReturn(rc, rc);
11036
11037 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11038 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11039 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11040 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11041
11042 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11043 {
11044 /*
11045 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11046 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11047 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11048 *
11049 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11050 */
11051 VMXDispatchHostNmi();
11052 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11053 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11054 return VINF_SUCCESS;
11055 }
11056
11057 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11058 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11059 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11060 { /* likely */ }
11061 else
11062 {
11063 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11064 rcStrictRc1 = VINF_SUCCESS;
11065 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11066 return rcStrictRc1;
11067 }
11068
11069 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11070 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11071 switch (uIntType)
11072 {
11073 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11074 Assert(uVector == X86_XCPT_DB);
11075 /* no break */
11076 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11077 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11078 /* no break */
11079 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11080 {
11081 switch (uVector)
11082 {
11083 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11084 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11085 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11086 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11087 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11088 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11089 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11090
11091 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11092 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11093 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11094 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11095 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11096 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11097 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11098 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11099 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11100 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11101 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11102 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11103 default:
11104 {
11105 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11106 AssertRCReturn(rc, rc);
11107
11108 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11109 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11110 {
11111 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11112 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11113 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11114
11115 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11116 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11117 AssertRCReturn(rc, rc);
11118 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11119 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11120 0 /* GCPtrFaultAddress */);
11121 AssertRCReturn(rc, rc);
11122 }
11123 else
11124 {
11125 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11126 pVCpu->hm.s.u32HMError = uVector;
11127 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11128 }
11129 break;
11130 }
11131 }
11132 break;
11133 }
11134
11135 default:
11136 {
11137 pVCpu->hm.s.u32HMError = uExitIntInfo;
11138 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11139 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11140 break;
11141 }
11142 }
11143 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11144 return rc;
11145}
11146
11147
11148/**
11149 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11150 */
11151HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11152{
11153 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11154
11155 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11156 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11157
11158 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11159 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11160 return VINF_SUCCESS;
11161}
11162
11163
11164/**
11165 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11166 */
11167HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11168{
11169 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11170 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11171 {
11172 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11173 HMVMX_RETURN_UNEXPECTED_EXIT();
11174 }
11175
11176 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11177
11178 /*
11179 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11180 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11181 */
11182 uint32_t uIntrState = 0;
11183 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11184 AssertRCReturn(rc, rc);
11185
11186 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11187 if ( fBlockSti
11188 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11189 {
11190 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11191 }
11192
11193 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11194 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11195
11196 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11197 return VINF_SUCCESS;
11198}
11199
11200
11201/**
11202 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11203 */
11204HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11205{
11206 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11207 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11208 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11209}
11210
11211
11212/**
11213 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11214 */
11215HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11216{
11217 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11218 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11219 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11220}
11221
11222
11223/**
11224 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11225 */
11226HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11227{
11228 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11229 PVM pVM = pVCpu->CTX_SUFF(pVM);
11230 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11231 if (RT_LIKELY(rc == VINF_SUCCESS))
11232 {
11233 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11234 Assert(pVmxTransient->cbInstr == 2);
11235 }
11236 else
11237 {
11238 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11239 rc = VERR_EM_INTERPRETER;
11240 }
11241 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11242 return rc;
11243}
11244
11245
11246/**
11247 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11248 */
11249HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11250{
11251 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11252 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11253 AssertRCReturn(rc, rc);
11254
11255 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11256 return VINF_EM_RAW_EMULATE_INSTR;
11257
11258 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11259 HMVMX_RETURN_UNEXPECTED_EXIT();
11260}
11261
11262
11263/**
11264 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11265 */
11266HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11267{
11268 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11269 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11270 AssertRCReturn(rc, rc);
11271
11272 PVM pVM = pVCpu->CTX_SUFF(pVM);
11273 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11274 if (RT_LIKELY(rc == VINF_SUCCESS))
11275 {
11276 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11277 Assert(pVmxTransient->cbInstr == 2);
11278 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11279 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11280 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11281 }
11282 else
11283 rc = VERR_EM_INTERPRETER;
11284 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11285 return rc;
11286}
11287
11288
11289/**
11290 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11291 */
11292HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11293{
11294 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11295 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11296 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11297 AssertRCReturn(rc, rc);
11298
11299 PVM pVM = pVCpu->CTX_SUFF(pVM);
11300 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11301 if (RT_SUCCESS(rc))
11302 {
11303 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11304 Assert(pVmxTransient->cbInstr == 3);
11305 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11306 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11307 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11308 }
11309 else
11310 {
11311 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11312 rc = VERR_EM_INTERPRETER;
11313 }
11314 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11315 return rc;
11316}
11317
11318
11319/**
11320 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11321 */
11322HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11323{
11324 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11325 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11326 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
11327 AssertRCReturn(rc, rc);
11328
11329 PVM pVM = pVCpu->CTX_SUFF(pVM);
11330 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11331 if (RT_LIKELY(rc == VINF_SUCCESS))
11332 {
11333 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11334 Assert(pVmxTransient->cbInstr == 2);
11335 }
11336 else
11337 {
11338 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11339 rc = VERR_EM_INTERPRETER;
11340 }
11341 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11342 return rc;
11343}
11344
11345
11346/**
11347 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11348 */
11349HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11350{
11351 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11352 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11353
11354 if (pVCpu->hm.s.fHypercallsEnabled)
11355 {
11356#if 0
11357 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11358#else
11359 /* Aggressive state sync. for now. */
11360 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11361 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11362#endif
11363 rc |= hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11364 AssertRCReturn(rc, rc);
11365
11366 /** @todo pre-increment RIP before hypercall will break when we have to implement
11367 * continuing hypercalls (e.g. Hyper-V). */
11368 /** @todo r=bird: GIMHypercall will probably have to be able to return
11369 * informational status codes, so it should be made VBOXSTRICTRC. Not
11370 * doing that now because the status code handling isn't clean (i.e.
11371 * if you use RT_SUCCESS(rc) on the result of something, you don't
11372 * return rc in the success case, you return VINF_SUCCESS). */
11373 rc = GIMHypercall(pVCpu, pMixedCtx);
11374 /* If the hypercall changes anything other than guest general-purpose registers,
11375 we would need to reload the guest changed bits here before VM-entry. */
11376 return rc;
11377 }
11378
11379 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11380 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11381 return VINF_SUCCESS;
11382}
11383
11384
11385/**
11386 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11387 */
11388HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11389{
11390 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11391 PVM pVM = pVCpu->CTX_SUFF(pVM);
11392 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11393
11394 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11395 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11396 AssertRCReturn(rc, rc);
11397
11398 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11399 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11400 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11401 else
11402 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11403 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11404 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11405 return rcStrict;
11406}
11407
11408
11409/**
11410 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11411 */
11412HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11413{
11414 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11415 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11416 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11417 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11418 AssertRCReturn(rc, rc);
11419
11420 PVM pVM = pVCpu->CTX_SUFF(pVM);
11421 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11422 if (RT_LIKELY(rc == VINF_SUCCESS))
11423 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11424 else
11425 {
11426 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11427 rc = VERR_EM_INTERPRETER;
11428 }
11429 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11430 return rc;
11431}
11432
11433
11434/**
11435 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11436 */
11437HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11438{
11439 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11440 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11441 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11442 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11443 AssertRCReturn(rc, rc);
11444
11445 PVM pVM = pVCpu->CTX_SUFF(pVM);
11446 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11447 rc = VBOXSTRICTRC_VAL(rc2);
11448 if (RT_LIKELY( rc == VINF_SUCCESS
11449 || rc == VINF_EM_HALT))
11450 {
11451 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11452 AssertRCReturn(rc3, rc3);
11453
11454 if ( rc == VINF_EM_HALT
11455 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11456 {
11457 rc = VINF_SUCCESS;
11458 }
11459 }
11460 else
11461 {
11462 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11463 rc = VERR_EM_INTERPRETER;
11464 }
11465 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11466 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11467 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11468 return rc;
11469}
11470
11471
11472/**
11473 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11474 */
11475HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11476{
11477 /*
11478 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11479 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11480 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11481 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11482 */
11483 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11484 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11485 HMVMX_RETURN_UNEXPECTED_EXIT();
11486}
11487
11488
11489/**
11490 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11491 */
11492HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11493{
11494 /*
11495 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11496 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11497 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11498 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11499 */
11500 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11501 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11502 HMVMX_RETURN_UNEXPECTED_EXIT();
11503}
11504
11505
11506/**
11507 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11508 */
11509HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11510{
11511 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11512 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11513 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11514 HMVMX_RETURN_UNEXPECTED_EXIT();
11515}
11516
11517
11518/**
11519 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11520 */
11521HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11522{
11523 /*
11524 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11525 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11526 * See Intel spec. 25.3 "Other Causes of VM-exits".
11527 */
11528 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11529 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11530 HMVMX_RETURN_UNEXPECTED_EXIT();
11531}
11532
11533
11534/**
11535 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11536 * VM-exit.
11537 */
11538HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11539{
11540 /*
11541 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11542 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11543 *
11544 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11545 * See Intel spec. "23.8 Restrictions on VMX operation".
11546 */
11547 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11548 return VINF_SUCCESS;
11549}
11550
11551
11552/**
11553 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11554 * VM-exit.
11555 */
11556HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11557{
11558 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11559 return VINF_EM_RESET;
11560}
11561
11562
11563/**
11564 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11565 */
11566HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11567{
11568 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11569 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11570 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11571 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11572 AssertRCReturn(rc, rc);
11573
11574 pMixedCtx->rip++;
11575 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11576 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11577 rc = VINF_SUCCESS;
11578 else
11579 rc = VINF_EM_HALT;
11580
11581 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11582 if (rc != VINF_SUCCESS)
11583 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11584 return rc;
11585}
11586
11587
11588/**
11589 * VM-exit handler for instructions that result in a \#UD exception delivered to
11590 * the guest.
11591 */
11592HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11593{
11594 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11595 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11596 return VINF_SUCCESS;
11597}
11598
11599
11600/**
11601 * VM-exit handler for expiry of the VMX preemption timer.
11602 */
11603HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11604{
11605 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11606
11607 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11608 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11609
11610 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11611 PVM pVM = pVCpu->CTX_SUFF(pVM);
11612 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11613 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11614 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11615}
11616
11617
11618/**
11619 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11620 */
11621HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11622{
11623 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11624
11625 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11626 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11627 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11628 AssertRCReturn(rc, rc);
11629
11630 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11631 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11632
11633 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11634
11635 return rcStrict;
11636}
11637
11638
11639/**
11640 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11641 */
11642HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11643{
11644 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11645
11646 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11647 /** @todo implement EMInterpretInvpcid() */
11648 return VERR_EM_INTERPRETER;
11649}
11650
11651
11652/**
11653 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11654 * Error VM-exit.
11655 */
11656HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11657{
11658 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11659 AssertRCReturn(rc, rc);
11660
11661 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11662 AssertRCReturn(rc, rc);
11663
11664 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11665 NOREF(uInvalidReason);
11666
11667#ifdef VBOX_STRICT
11668 uint32_t uIntrState;
11669 RTHCUINTREG uHCReg;
11670 uint64_t u64Val;
11671 uint32_t u32Val;
11672
11673 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11674 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11675 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11676 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11677 AssertRCReturn(rc, rc);
11678
11679 Log4(("uInvalidReason %u\n", uInvalidReason));
11680 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11681 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11682 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11683 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11684
11685 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11686 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11687 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11688 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11689 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11690 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11691 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11692 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11693 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11694 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11695 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11696 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11697#else
11698 NOREF(pVmxTransient);
11699#endif
11700
11701 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11702 return VERR_VMX_INVALID_GUEST_STATE;
11703}
11704
11705
11706/**
11707 * VM-exit handler for VM-entry failure due to an MSR-load
11708 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11709 */
11710HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11711{
11712 NOREF(pVmxTransient);
11713 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11714 HMVMX_RETURN_UNEXPECTED_EXIT();
11715}
11716
11717
11718/**
11719 * VM-exit handler for VM-entry failure due to a machine-check event
11720 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11721 */
11722HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11723{
11724 NOREF(pVmxTransient);
11725 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11726 HMVMX_RETURN_UNEXPECTED_EXIT();
11727}
11728
11729
11730/**
11731 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11732 * theory.
11733 */
11734HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11735{
11736 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11737 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11738 return VERR_VMX_UNDEFINED_EXIT_CODE;
11739}
11740
11741
11742/**
11743 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11744 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11745 * Conditional VM-exit.
11746 */
11747HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11748{
11749 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11750
11751 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11752 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11753 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11754 return VERR_EM_INTERPRETER;
11755 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11756 HMVMX_RETURN_UNEXPECTED_EXIT();
11757}
11758
11759
11760/**
11761 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11762 */
11763HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11764{
11765 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11766
11767 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11768 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11769 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11770 return VERR_EM_INTERPRETER;
11771 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11772 HMVMX_RETURN_UNEXPECTED_EXIT();
11773}
11774
11775
11776/**
11777 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11778 */
11779HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11780{
11781 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11782
11783 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11784 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11785 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11786 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11787 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11788 {
11789 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11790 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11791 }
11792 AssertRCReturn(rc, rc);
11793 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11794
11795#ifdef VBOX_STRICT
11796 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11797 {
11798 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11799 && pMixedCtx->ecx != MSR_K6_EFER)
11800 {
11801 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11802 pMixedCtx->ecx));
11803 HMVMX_RETURN_UNEXPECTED_EXIT();
11804 }
11805# if HC_ARCH_BITS == 64
11806 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
11807 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11808 {
11809 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11810 HMVMX_RETURN_UNEXPECTED_EXIT();
11811 }
11812# endif
11813 }
11814#endif
11815
11816 PVM pVM = pVCpu->CTX_SUFF(pVM);
11817 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11818 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11819 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11820 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11821 if (RT_SUCCESS(rc))
11822 {
11823 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11824 Assert(pVmxTransient->cbInstr == 2);
11825 }
11826 return rc;
11827}
11828
11829
11830/**
11831 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11832 */
11833HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11834{
11835 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11836 PVM pVM = pVCpu->CTX_SUFF(pVM);
11837 int rc = VINF_SUCCESS;
11838
11839 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11840 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11841 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11842 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11843 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11844 {
11845 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11846 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11847 }
11848 AssertRCReturn(rc, rc);
11849 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11850
11851 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11852 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11853 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11854
11855 if (RT_SUCCESS(rc))
11856 {
11857 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11858
11859 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11860 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11861 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
11862 {
11863 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11864 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11865 EMInterpretWrmsr() changes it. */
11866 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11867 }
11868 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11869 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11870 else if (pMixedCtx->ecx == MSR_K6_EFER)
11871 {
11872 /*
11873 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11874 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11875 * the other bits as well, SCE and NXE. See @bugref{7368}.
11876 */
11877 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
11878 }
11879
11880 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11881 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11882 {
11883 switch (pMixedCtx->ecx)
11884 {
11885 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11886 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11887 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11888 case MSR_K8_FS_BASE: /* no break */
11889 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
11890 case MSR_K6_EFER: /* already handled above */ break;
11891 default:
11892 {
11893 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11894 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11895#if HC_ARCH_BITS == 64
11896 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11897 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
11898#endif
11899 break;
11900 }
11901 }
11902 }
11903#ifdef VBOX_STRICT
11904 else
11905 {
11906 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11907 switch (pMixedCtx->ecx)
11908 {
11909 case MSR_IA32_SYSENTER_CS:
11910 case MSR_IA32_SYSENTER_EIP:
11911 case MSR_IA32_SYSENTER_ESP:
11912 case MSR_K8_FS_BASE:
11913 case MSR_K8_GS_BASE:
11914 {
11915 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11916 HMVMX_RETURN_UNEXPECTED_EXIT();
11917 }
11918
11919 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11920 default:
11921 {
11922 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11923 {
11924 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
11925 if (pMixedCtx->ecx != MSR_K6_EFER)
11926 {
11927 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11928 pMixedCtx->ecx));
11929 HMVMX_RETURN_UNEXPECTED_EXIT();
11930 }
11931 }
11932
11933#if HC_ARCH_BITS == 64
11934 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11935 {
11936 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11937 HMVMX_RETURN_UNEXPECTED_EXIT();
11938 }
11939#endif
11940 break;
11941 }
11942 }
11943 }
11944#endif /* VBOX_STRICT */
11945 }
11946 return rc;
11947}
11948
11949
11950/**
11951 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11952 */
11953HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11954{
11955 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11956
11957 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
11958 return VINF_EM_RAW_INTERRUPT;
11959}
11960
11961
11962/**
11963 * VM-exit handler for when the TPR value is lowered below the specified
11964 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11965 */
11966HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11967{
11968 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11969 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11970
11971 /*
11972 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
11973 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
11974 * resume guest execution.
11975 */
11976 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11977 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11978 return VINF_SUCCESS;
11979}
11980
11981
11982/**
11983 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11984 * VM-exit.
11985 *
11986 * @retval VINF_SUCCESS when guest execution can continue.
11987 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11988 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11989 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11990 * interpreter.
11991 */
11992HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11993{
11994 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11995 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11996 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11997 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11998 AssertRCReturn(rc, rc);
11999
12000 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12001 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12002 PVM pVM = pVCpu->CTX_SUFF(pVM);
12003 VBOXSTRICTRC rcStrict;
12004 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12005 switch (uAccessType)
12006 {
12007 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12008 {
12009 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12010 AssertRCReturn(rc, rc);
12011
12012 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12013 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12014 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12015 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12016 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12017 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12018 {
12019 case 0: /* CR0 */
12020 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12021 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12022 break;
12023 case 2: /* CR2 */
12024 /* Nothing to do here, CR2 it's not part of the VMCS. */
12025 break;
12026 case 3: /* CR3 */
12027 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12028 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12029 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12030 break;
12031 case 4: /* CR4 */
12032 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12033 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12034 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12035 break;
12036 case 8: /* CR8 */
12037 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12038 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12039 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12040 break;
12041 default:
12042 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12043 break;
12044 }
12045
12046 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12047 break;
12048 }
12049
12050 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12051 {
12052 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12053 AssertRCReturn(rc, rc);
12054
12055 Assert( !pVM->hm.s.fNestedPaging
12056 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12057 || pVCpu->hm.s.fUsingDebugLoop
12058 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12059
12060 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12061 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12062 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12063
12064 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12065 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12066 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12067 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12068 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12069 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12070 VBOXSTRICTRC_VAL(rcStrict)));
12071 break;
12072 }
12073
12074 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12075 {
12076 AssertRCReturn(rc, rc);
12077 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12078 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12079 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12080 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12081 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12082 break;
12083 }
12084
12085 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12086 {
12087 AssertRCReturn(rc, rc);
12088 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12089 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12090 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12091 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12092 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12093 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12094 break;
12095 }
12096
12097 default:
12098 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12099 VERR_VMX_UNEXPECTED_EXCEPTION);
12100 }
12101
12102 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12103 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12104 NOREF(pVM);
12105 return rcStrict;
12106}
12107
12108
12109/**
12110 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12111 * VM-exit.
12112 */
12113HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12114{
12115 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12116 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12117
12118 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12119 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12120 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12121 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12122 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12123 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12124 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12125 AssertRCReturn(rc2, rc2);
12126
12127 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12128 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12129 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12130 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12131 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12132 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12133 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12134 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12135 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12136
12137 /* I/O operation lookup arrays. */
12138 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12139 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12140
12141 VBOXSTRICTRC rcStrict;
12142 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12143 uint32_t const cbInstr = pVmxTransient->cbInstr;
12144 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12145 PVM pVM = pVCpu->CTX_SUFF(pVM);
12146 if (fIOString)
12147 {
12148#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12149 See @bugref{5752#c158}. Should work now. */
12150 /*
12151 * INS/OUTS - I/O String instruction.
12152 *
12153 * Use instruction-information if available, otherwise fall back on
12154 * interpreting the instruction.
12155 */
12156 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12157 fIOWrite ? 'w' : 'r'));
12158 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12159 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12160 {
12161 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12162 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12163 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12164 AssertRCReturn(rc2, rc2);
12165 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12166 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12167 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12168 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12169 if (fIOWrite)
12170 {
12171 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12172 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
12173 }
12174 else
12175 {
12176 /*
12177 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12178 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12179 * See Intel Instruction spec. for "INS".
12180 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12181 */
12182 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
12183 }
12184 }
12185 else
12186 {
12187 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12188 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12189 AssertRCReturn(rc2, rc2);
12190 rcStrict = IEMExecOne(pVCpu);
12191 }
12192 /** @todo IEM needs to be setting these flags somehow. */
12193 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12194 fUpdateRipAlready = true;
12195#else
12196 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12197 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12198 if (RT_SUCCESS(rcStrict))
12199 {
12200 if (fIOWrite)
12201 {
12202 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12203 (DISCPUMODE)pDis->uAddrMode, cbValue);
12204 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12205 }
12206 else
12207 {
12208 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12209 (DISCPUMODE)pDis->uAddrMode, cbValue);
12210 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12211 }
12212 }
12213 else
12214 {
12215 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12216 pMixedCtx->rip));
12217 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12218 }
12219#endif
12220 }
12221 else
12222 {
12223 /*
12224 * IN/OUT - I/O instruction.
12225 */
12226 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12227 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12228 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12229 if (fIOWrite)
12230 {
12231 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12232 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
12233 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12235 }
12236 else
12237 {
12238 uint32_t u32Result = 0;
12239 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12240 if (IOM_SUCCESS(rcStrict))
12241 {
12242 /* Save result of I/O IN instr. in AL/AX/EAX. */
12243 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12244 }
12245 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12246 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12247 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12248 }
12249 }
12250
12251 if (IOM_SUCCESS(rcStrict))
12252 {
12253 if (!fUpdateRipAlready)
12254 {
12255 pMixedCtx->rip += cbInstr;
12256 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12257 }
12258
12259 /*
12260 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12261 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12262 */
12263 if (fIOString)
12264 {
12265 /** @todo Single-step for INS/OUTS with REP prefix? */
12266 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12267 }
12268 else if ( !fDbgStepping
12269 && fGstStepping)
12270 {
12271 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12272 }
12273
12274 /*
12275 * If any I/O breakpoints are armed, we need to check if one triggered
12276 * and take appropriate action.
12277 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12278 */
12279 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12280 AssertRCReturn(rc2, rc2);
12281
12282 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12283 * execution engines about whether hyper BPs and such are pending. */
12284 uint32_t const uDr7 = pMixedCtx->dr[7];
12285 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12286 && X86_DR7_ANY_RW_IO(uDr7)
12287 && (pMixedCtx->cr4 & X86_CR4_DE))
12288 || DBGFBpIsHwIoArmed(pVM)))
12289 {
12290 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12291
12292 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12293 VMMRZCallRing3Disable(pVCpu);
12294 HM_DISABLE_PREEMPT();
12295
12296 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12297
12298 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12299 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12300 {
12301 /* Raise #DB. */
12302 if (fIsGuestDbgActive)
12303 ASMSetDR6(pMixedCtx->dr[6]);
12304 if (pMixedCtx->dr[7] != uDr7)
12305 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12306
12307 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12308 }
12309 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
12310 else if ( rcStrict2 != VINF_SUCCESS
12311 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12312 rcStrict = rcStrict2;
12313
12314 HM_RESTORE_PREEMPT();
12315 VMMRZCallRing3Enable(pVCpu);
12316 }
12317 }
12318
12319#ifdef VBOX_STRICT
12320 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12321 Assert(!fIOWrite);
12322 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
12323 Assert(fIOWrite);
12324 else
12325 {
12326#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12327 * statuses, that the VMM device and some others may return. See
12328 * IOM_SUCCESS() for guidance. */
12329 AssertMsg( RT_FAILURE(rcStrict)
12330 || rcStrict == VINF_SUCCESS
12331 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12332 || rcStrict == VINF_EM_DBG_BREAKPOINT
12333 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12334 || rcStrict == VINF_EM_RAW_TO_R3
12335 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12336#endif
12337 }
12338#endif
12339
12340 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12341 return rcStrict;
12342}
12343
12344
12345/**
12346 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12347 * VM-exit.
12348 */
12349HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12350{
12351 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12352
12353 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12354 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12355 AssertRCReturn(rc, rc);
12356 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12357 {
12358 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12359 AssertRCReturn(rc, rc);
12360 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12361 {
12362 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12363
12364 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12365 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12366
12367 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12368 Assert(!pVCpu->hm.s.Event.fPending);
12369 pVCpu->hm.s.Event.fPending = true;
12370 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12371 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12372 AssertRCReturn(rc, rc);
12373 if (fErrorCodeValid)
12374 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12375 else
12376 pVCpu->hm.s.Event.u32ErrCode = 0;
12377 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12378 && uVector == X86_XCPT_PF)
12379 {
12380 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12381 }
12382
12383 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12384 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12385 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12386 }
12387 }
12388
12389 /** @todo Emulate task switch someday, currently just going back to ring-3 for
12390 * emulation. */
12391 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12392 return VERR_EM_INTERPRETER;
12393}
12394
12395
12396/**
12397 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12398 */
12399HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12400{
12401 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12402 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12403 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12404 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12405 AssertRCReturn(rc, rc);
12406 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12407 return VINF_EM_DBG_STEPPED;
12408}
12409
12410
12411/**
12412 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12413 */
12414HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12415{
12416 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12417
12418 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12419 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12420 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12421 { /* likely */ }
12422 else
12423 {
12424 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12425 rcStrict1 = VINF_SUCCESS;
12426 return rcStrict1;
12427 }
12428
12429#if 0
12430 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12431 * just sync the whole thing. */
12432 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12433#else
12434 /* Aggressive state sync. for now. */
12435 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12436 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12437 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12438#endif
12439 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12440 AssertRCReturn(rc, rc);
12441
12442 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12443 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12444 VBOXSTRICTRC rcStrict2;
12445 switch (uAccessType)
12446 {
12447 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12448 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12449 {
12450 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12451 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12452 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12453
12454 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12455 GCPhys &= PAGE_BASE_GC_MASK;
12456 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12457 PVM pVM = pVCpu->CTX_SUFF(pVM);
12458 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12459 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12460
12461 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12462 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12463 CPUMCTX2CORE(pMixedCtx), GCPhys);
12464 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12465 if ( rcStrict2 == VINF_SUCCESS
12466 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12467 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12468 {
12469 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12470 | HM_CHANGED_GUEST_RSP
12471 | HM_CHANGED_GUEST_RFLAGS
12472 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12473 rcStrict2 = VINF_SUCCESS;
12474 }
12475 break;
12476 }
12477
12478 default:
12479 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12480 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12481 break;
12482 }
12483
12484 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12485 if (rcStrict2 != VINF_SUCCESS)
12486 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12487 return rcStrict2;
12488}
12489
12490
12491/**
12492 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12493 * VM-exit.
12494 */
12495HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12496{
12497 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12498
12499 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12500 if (pVmxTransient->fWasGuestDebugStateActive)
12501 {
12502 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12503 HMVMX_RETURN_UNEXPECTED_EXIT();
12504 }
12505
12506 if ( !pVCpu->hm.s.fSingleInstruction
12507 && !pVmxTransient->fWasHyperDebugStateActive)
12508 {
12509 Assert(!DBGFIsStepping(pVCpu));
12510 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12511
12512 /* Don't intercept MOV DRx any more. */
12513 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12514 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12515 AssertRCReturn(rc, rc);
12516
12517 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12518 VMMRZCallRing3Disable(pVCpu);
12519 HM_DISABLE_PREEMPT();
12520
12521 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12522 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12523 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12524
12525 HM_RESTORE_PREEMPT();
12526 VMMRZCallRing3Enable(pVCpu);
12527
12528#ifdef VBOX_WITH_STATISTICS
12529 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12530 AssertRCReturn(rc, rc);
12531 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12532 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12533 else
12534 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12535#endif
12536 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12537 return VINF_SUCCESS;
12538 }
12539
12540 /*
12541 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12542 * Update the segment registers and DR7 from the CPU.
12543 */
12544 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12545 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12546 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12547 AssertRCReturn(rc, rc);
12548 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12549
12550 PVM pVM = pVCpu->CTX_SUFF(pVM);
12551 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12552 {
12553 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12554 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12555 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12556 if (RT_SUCCESS(rc))
12557 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12558 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12559 }
12560 else
12561 {
12562 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12563 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12564 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12565 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12566 }
12567
12568 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12569 if (RT_SUCCESS(rc))
12570 {
12571 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12572 AssertRCReturn(rc2, rc2);
12573 return VINF_SUCCESS;
12574 }
12575 return rc;
12576}
12577
12578
12579/**
12580 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12581 * Conditional VM-exit.
12582 */
12583HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12584{
12585 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12586 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12587
12588 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12589 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12590 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12591 { /* likely */ }
12592 else
12593 {
12594 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12595 rcStrict1 = VINF_SUCCESS;
12596 return rcStrict1;
12597 }
12598
12599 RTGCPHYS GCPhys = 0;
12600 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12601
12602#if 0
12603 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12604#else
12605 /* Aggressive state sync. for now. */
12606 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12607 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12608 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12609#endif
12610 AssertRCReturn(rc, rc);
12611
12612 /*
12613 * If we succeed, resume guest execution.
12614 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12615 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12616 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12617 * weird case. See @bugref{6043}.
12618 */
12619 PVM pVM = pVCpu->CTX_SUFF(pVM);
12620 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12621 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12622 if ( rcStrict2 == VINF_SUCCESS
12623 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12624 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12625 {
12626 /* Successfully handled MMIO operation. */
12627 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12628 | HM_CHANGED_GUEST_RSP
12629 | HM_CHANGED_GUEST_RFLAGS
12630 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12631 return VINF_SUCCESS;
12632 }
12633 return rcStrict2;
12634}
12635
12636
12637/**
12638 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12639 * VM-exit.
12640 */
12641HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12642{
12643 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12644 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12645
12646 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12647 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12648 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12649 { /* likely */ }
12650 else
12651 {
12652 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12653 rcStrict1 = VINF_SUCCESS;
12654 return rcStrict1;
12655 }
12656
12657 RTGCPHYS GCPhys = 0;
12658 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12659 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12660#if 0
12661 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12662#else
12663 /* Aggressive state sync. for now. */
12664 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12665 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12666 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12667#endif
12668 AssertRCReturn(rc, rc);
12669
12670 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12671 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12672
12673 RTGCUINT uErrorCode = 0;
12674 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12675 uErrorCode |= X86_TRAP_PF_ID;
12676 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12677 uErrorCode |= X86_TRAP_PF_RW;
12678 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12679 uErrorCode |= X86_TRAP_PF_P;
12680
12681 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12682
12683 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12684 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12685
12686 /* Handle the pagefault trap for the nested shadow table. */
12687 PVM pVM = pVCpu->CTX_SUFF(pVM);
12688 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12689 TRPMResetTrap(pVCpu);
12690
12691 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12692 if ( rcStrict2 == VINF_SUCCESS
12693 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12694 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12695 {
12696 /* Successfully synced our nested page tables. */
12697 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12698 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12699 | HM_CHANGED_GUEST_RSP
12700 | HM_CHANGED_GUEST_RFLAGS);
12701 return VINF_SUCCESS;
12702 }
12703
12704 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12705 return rcStrict2;
12706}
12707
12708/** @} */
12709
12710/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12711/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12712/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12713
12714/** @name VM-exit exception handlers.
12715 * @{
12716 */
12717
12718/**
12719 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12720 */
12721static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12722{
12723 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12724 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12725
12726 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12727 AssertRCReturn(rc, rc);
12728
12729 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12730 {
12731 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12732 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12733
12734 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12735 * provides VM-exit instruction length. If this causes problem later,
12736 * disassemble the instruction like it's done on AMD-V. */
12737 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12738 AssertRCReturn(rc2, rc2);
12739 return rc;
12740 }
12741
12742 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12743 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12744 return rc;
12745}
12746
12747
12748/**
12749 * VM-exit exception handler for \#BP (Breakpoint exception).
12750 */
12751static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12752{
12753 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12754 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12755
12756 /** @todo Try optimize this by not saving the entire guest state unless
12757 * really needed. */
12758 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12759 AssertRCReturn(rc, rc);
12760
12761 PVM pVM = pVCpu->CTX_SUFF(pVM);
12762 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12763 if (rc == VINF_EM_RAW_GUEST_TRAP)
12764 {
12765 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12766 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12767 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12768 AssertRCReturn(rc, rc);
12769
12770 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12771 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12772 }
12773
12774 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12775 return rc;
12776}
12777
12778
12779/**
12780 * VM-exit exception handler for \#AC (alignment check exception).
12781 */
12782static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12783{
12784 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12785
12786 /*
12787 * Re-inject it. We'll detect any nesting before getting here.
12788 */
12789 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12790 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12791 AssertRCReturn(rc, rc);
12792 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12793
12794 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12795 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12796 return VINF_SUCCESS;
12797}
12798
12799
12800/**
12801 * VM-exit exception handler for \#DB (Debug exception).
12802 */
12803static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12804{
12805 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12806 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12807 Log6(("XcptDB\n"));
12808
12809 /*
12810 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12811 * for processing.
12812 */
12813 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12814 AssertRCReturn(rc, rc);
12815
12816 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12817 uint64_t uDR6 = X86_DR6_INIT_VAL;
12818 uDR6 |= ( pVmxTransient->uExitQualification
12819 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12820
12821 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12822 if (rc == VINF_EM_RAW_GUEST_TRAP)
12823 {
12824 /*
12825 * The exception was for the guest. Update DR6, DR7.GD and
12826 * IA32_DEBUGCTL.LBR before forwarding it.
12827 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12828 */
12829 VMMRZCallRing3Disable(pVCpu);
12830 HM_DISABLE_PREEMPT();
12831
12832 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12833 pMixedCtx->dr[6] |= uDR6;
12834 if (CPUMIsGuestDebugStateActive(pVCpu))
12835 ASMSetDR6(pMixedCtx->dr[6]);
12836
12837 HM_RESTORE_PREEMPT();
12838 VMMRZCallRing3Enable(pVCpu);
12839
12840 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12841 AssertRCReturn(rc, rc);
12842
12843 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12844 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12845
12846 /* Paranoia. */
12847 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12848 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12849
12850 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12851 AssertRCReturn(rc, rc);
12852
12853 /*
12854 * Raise #DB in the guest.
12855 *
12856 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
12857 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
12858 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
12859 *
12860 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
12861 */
12862 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12863 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12864 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12865 AssertRCReturn(rc, rc);
12866 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12867 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12868 return VINF_SUCCESS;
12869 }
12870
12871 /*
12872 * Not a guest trap, must be a hypervisor related debug event then.
12873 * Update DR6 in case someone is interested in it.
12874 */
12875 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12876 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12877 CPUMSetHyperDR6(pVCpu, uDR6);
12878
12879 return rc;
12880}
12881
12882
12883/**
12884 * VM-exit exception handler for \#NM (Device-not-available exception: floating
12885 * point exception).
12886 */
12887static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12888{
12889 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12890
12891 /* We require CR0 and EFER. EFER is always up-to-date. */
12892 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12893 AssertRCReturn(rc, rc);
12894
12895 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
12896 VMMRZCallRing3Disable(pVCpu);
12897 HM_DISABLE_PREEMPT();
12898
12899 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
12900 if (pVmxTransient->fWasGuestFPUStateActive)
12901 {
12902 rc = VINF_EM_RAW_GUEST_TRAP;
12903 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
12904 }
12905 else
12906 {
12907#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12908 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
12909#endif
12910 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12911 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
12912 }
12913
12914 HM_RESTORE_PREEMPT();
12915 VMMRZCallRing3Enable(pVCpu);
12916
12917 if (rc == VINF_SUCCESS)
12918 {
12919 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
12920 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12921 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
12922 pVCpu->hm.s.fPreloadGuestFpu = true;
12923 }
12924 else
12925 {
12926 /* Forward #NM to the guest. */
12927 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
12928 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12929 AssertRCReturn(rc, rc);
12930 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12931 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
12932 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12933 }
12934
12935 return VINF_SUCCESS;
12936}
12937
12938
12939/**
12940 * VM-exit exception handler for \#GP (General-protection exception).
12941 *
12942 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12943 */
12944static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12945{
12946 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12947 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12948
12949 int rc;
12950 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12951 { /* likely */ }
12952 else
12953 {
12954#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12955 Assert(pVCpu->hm.s.fUsingDebugLoop);
12956#endif
12957 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12958 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12959 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12960 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12961 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12962 AssertRCReturn(rc, rc);
12963 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12964 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12965 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12966 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12967 return rc;
12968 }
12969
12970 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12971 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12972
12973 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12974 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12975 AssertRCReturn(rc, rc);
12976
12977 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12978 uint32_t cbOp = 0;
12979 PVM pVM = pVCpu->CTX_SUFF(pVM);
12980 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12981 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12982 if (RT_SUCCESS(rc))
12983 {
12984 rc = VINF_SUCCESS;
12985 Assert(cbOp == pDis->cbInstr);
12986 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12987 switch (pDis->pCurInstr->uOpcode)
12988 {
12989 case OP_CLI:
12990 {
12991 pMixedCtx->eflags.Bits.u1IF = 0;
12992 pMixedCtx->eflags.Bits.u1RF = 0;
12993 pMixedCtx->rip += pDis->cbInstr;
12994 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12995 if ( !fDbgStepping
12996 && pMixedCtx->eflags.Bits.u1TF)
12997 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12998 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12999 break;
13000 }
13001
13002 case OP_STI:
13003 {
13004 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13005 pMixedCtx->eflags.Bits.u1IF = 1;
13006 pMixedCtx->eflags.Bits.u1RF = 0;
13007 pMixedCtx->rip += pDis->cbInstr;
13008 if (!fOldIF)
13009 {
13010 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13011 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13012 }
13013 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13014 if ( !fDbgStepping
13015 && pMixedCtx->eflags.Bits.u1TF)
13016 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13017 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13018 break;
13019 }
13020
13021 case OP_HLT:
13022 {
13023 rc = VINF_EM_HALT;
13024 pMixedCtx->rip += pDis->cbInstr;
13025 pMixedCtx->eflags.Bits.u1RF = 0;
13026 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13027 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13028 break;
13029 }
13030
13031 case OP_POPF:
13032 {
13033 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13034 uint32_t cbParm;
13035 uint32_t uMask;
13036 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13037 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13038 {
13039 cbParm = 4;
13040 uMask = 0xffffffff;
13041 }
13042 else
13043 {
13044 cbParm = 2;
13045 uMask = 0xffff;
13046 }
13047
13048 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13049 RTGCPTR GCPtrStack = 0;
13050 X86EFLAGS Eflags;
13051 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13052 &GCPtrStack);
13053 if (RT_SUCCESS(rc))
13054 {
13055 Assert(sizeof(Eflags.u32) >= cbParm);
13056 Eflags.u32 = 0;
13057 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13058 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13059 }
13060 if (RT_FAILURE(rc))
13061 {
13062 rc = VERR_EM_INTERPRETER;
13063 break;
13064 }
13065 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13066 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13067 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13068 pMixedCtx->esp += cbParm;
13069 pMixedCtx->esp &= uMask;
13070 pMixedCtx->rip += pDis->cbInstr;
13071 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13072 | HM_CHANGED_GUEST_RSP
13073 | HM_CHANGED_GUEST_RFLAGS);
13074 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13075 POPF restores EFLAGS.TF. */
13076 if ( !fDbgStepping
13077 && fGstStepping)
13078 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13079 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13080 break;
13081 }
13082
13083 case OP_PUSHF:
13084 {
13085 uint32_t cbParm;
13086 uint32_t uMask;
13087 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13088 {
13089 cbParm = 4;
13090 uMask = 0xffffffff;
13091 }
13092 else
13093 {
13094 cbParm = 2;
13095 uMask = 0xffff;
13096 }
13097
13098 /* Get the stack pointer & push the contents of eflags onto the stack. */
13099 RTGCPTR GCPtrStack = 0;
13100 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13101 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13102 if (RT_FAILURE(rc))
13103 {
13104 rc = VERR_EM_INTERPRETER;
13105 break;
13106 }
13107 X86EFLAGS Eflags = pMixedCtx->eflags;
13108 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13109 Eflags.Bits.u1RF = 0;
13110 Eflags.Bits.u1VM = 0;
13111
13112 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13113 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13114 {
13115 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13116 rc = VERR_EM_INTERPRETER;
13117 break;
13118 }
13119 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13120 pMixedCtx->esp -= cbParm;
13121 pMixedCtx->esp &= uMask;
13122 pMixedCtx->rip += pDis->cbInstr;
13123 pMixedCtx->eflags.Bits.u1RF = 0;
13124 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13125 | HM_CHANGED_GUEST_RSP
13126 | HM_CHANGED_GUEST_RFLAGS);
13127 if ( !fDbgStepping
13128 && pMixedCtx->eflags.Bits.u1TF)
13129 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13130 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13131 break;
13132 }
13133
13134 case OP_IRET:
13135 {
13136 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13137 * instruction reference. */
13138 RTGCPTR GCPtrStack = 0;
13139 uint32_t uMask = 0xffff;
13140 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13141 uint16_t aIretFrame[3];
13142 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13143 {
13144 rc = VERR_EM_INTERPRETER;
13145 break;
13146 }
13147 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13148 &GCPtrStack);
13149 if (RT_SUCCESS(rc))
13150 {
13151 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13152 PGMACCESSORIGIN_HM));
13153 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13154 }
13155 if (RT_FAILURE(rc))
13156 {
13157 rc = VERR_EM_INTERPRETER;
13158 break;
13159 }
13160 pMixedCtx->eip = 0;
13161 pMixedCtx->ip = aIretFrame[0];
13162 pMixedCtx->cs.Sel = aIretFrame[1];
13163 pMixedCtx->cs.ValidSel = aIretFrame[1];
13164 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13165 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13166 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13167 pMixedCtx->sp += sizeof(aIretFrame);
13168 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13169 | HM_CHANGED_GUEST_SEGMENT_REGS
13170 | HM_CHANGED_GUEST_RSP
13171 | HM_CHANGED_GUEST_RFLAGS);
13172 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13173 if ( !fDbgStepping
13174 && fGstStepping)
13175 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13176 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13177 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13178 break;
13179 }
13180
13181 case OP_INT:
13182 {
13183 uint16_t uVector = pDis->Param1.uValue & 0xff;
13184 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13185 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13186 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13187 break;
13188 }
13189
13190 case OP_INTO:
13191 {
13192 if (pMixedCtx->eflags.Bits.u1OF)
13193 {
13194 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13195 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13196 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13197 }
13198 else
13199 {
13200 pMixedCtx->eflags.Bits.u1RF = 0;
13201 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13202 }
13203 break;
13204 }
13205
13206 default:
13207 {
13208 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13209 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13210 EMCODETYPE_SUPERVISOR);
13211 rc = VBOXSTRICTRC_VAL(rc2);
13212 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13213 /** @todo We have to set pending-debug exceptions here when the guest is
13214 * single-stepping depending on the instruction that was interpreted. */
13215 Log4(("#GP rc=%Rrc\n", rc));
13216 break;
13217 }
13218 }
13219 }
13220 else
13221 rc = VERR_EM_INTERPRETER;
13222
13223 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13224 ("#GP Unexpected rc=%Rrc\n", rc));
13225 return rc;
13226}
13227
13228
13229/**
13230 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13231 * the exception reported in the VMX transient structure back into the VM.
13232 *
13233 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13234 * up-to-date.
13235 */
13236static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13237{
13238 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13239#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13240 Assert(pVCpu->hm.s.fUsingDebugLoop);
13241#endif
13242
13243 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13244 hmR0VmxCheckExitDueToEventDelivery(). */
13245 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13246 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13247 AssertRCReturn(rc, rc);
13248 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13249
13250#ifdef DEBUG_ramshankar
13251 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13252 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13253 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13254#endif
13255
13256 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13257 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13258 return VINF_SUCCESS;
13259}
13260
13261
13262/**
13263 * VM-exit exception handler for \#PF (Page-fault exception).
13264 */
13265static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13266{
13267 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13268 PVM pVM = pVCpu->CTX_SUFF(pVM);
13269 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13270 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13271 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13272 AssertRCReturn(rc, rc);
13273
13274 if (!pVM->hm.s.fNestedPaging)
13275 { /* likely */ }
13276 else
13277 {
13278#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13279 Assert(pVCpu->hm.s.fUsingDebugLoop);
13280#endif
13281 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13282 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13283 {
13284 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13285 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13286 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13287 }
13288 else
13289 {
13290 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13291 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13292 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13293 }
13294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13295 return rc;
13296 }
13297
13298 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13299 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13300 if (pVmxTransient->fVectoringPF)
13301 {
13302 Assert(pVCpu->hm.s.Event.fPending);
13303 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13304 }
13305
13306 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13307 AssertRCReturn(rc, rc);
13308
13309 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13310 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13311
13312 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13313 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13314 (RTGCPTR)pVmxTransient->uExitQualification);
13315
13316 Log4(("#PF: rc=%Rrc\n", rc));
13317 if (rc == VINF_SUCCESS)
13318 {
13319 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13320 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13321 * memory? We don't update the whole state here... */
13322 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13323 | HM_CHANGED_GUEST_RSP
13324 | HM_CHANGED_GUEST_RFLAGS
13325 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13326 TRPMResetTrap(pVCpu);
13327 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13328 return rc;
13329 }
13330
13331 if (rc == VINF_EM_RAW_GUEST_TRAP)
13332 {
13333 if (!pVmxTransient->fVectoringDoublePF)
13334 {
13335 /* It's a guest page fault and needs to be reflected to the guest. */
13336 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13337 TRPMResetTrap(pVCpu);
13338 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13339 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13340 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13341 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13342 }
13343 else
13344 {
13345 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13346 TRPMResetTrap(pVCpu);
13347 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13348 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13349 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13350 }
13351
13352 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13353 return VINF_SUCCESS;
13354 }
13355
13356 TRPMResetTrap(pVCpu);
13357 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13358 return rc;
13359}
13360
13361/** @} */
13362
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette