VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 62478

Last change on this file since 62478 was 62478, checked in by vboxsync, 9 years ago

(C) 2016
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 583.4 KB
Line 
1/* $Id: HMVMXR0.cpp 62478 2016-07-22 18:29:06Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#ifdef VBOX_WITH_NEW_APIC
38# include <VBox/vmm/apic.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_SWAP_FPU_STATE
53# define HMVMX_ALWAYS_FLUSH_TLB
54# define HMVMX_ALWAYS_SWAP_EFER
55#endif
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61/** Use the function table. */
62#define HMVMX_USE_FUNCTION_TABLE
63
64/** Determine which tagged-TLB flush handler to use. */
65#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70/** @name Updated-guest-state flags.
71 * @{ */
72#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
85#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
86#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
87#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
88#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
89#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
90#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
91#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93 | HMVMX_UPDATED_GUEST_RSP \
94 | HMVMX_UPDATED_GUEST_RFLAGS \
95 | HMVMX_UPDATED_GUEST_CR0 \
96 | HMVMX_UPDATED_GUEST_CR3 \
97 | HMVMX_UPDATED_GUEST_CR4 \
98 | HMVMX_UPDATED_GUEST_GDTR \
99 | HMVMX_UPDATED_GUEST_IDTR \
100 | HMVMX_UPDATED_GUEST_LDTR \
101 | HMVMX_UPDATED_GUEST_TR \
102 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103 | HMVMX_UPDATED_GUEST_DEBUG \
104 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
109 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110 | HMVMX_UPDATED_GUEST_INTR_STATE \
111 | HMVMX_UPDATED_GUEST_APIC_STATE)
112/** @} */
113
114/** @name
115 * Flags to skip redundant reads of some common VMCS fields that are not part of
116 * the guest-CPU state but are in the transient structure.
117 */
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually except:
143 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
144 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
145 * due to bugs in Intel CPUs.
146 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
147 * support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*********************************************************************************************************************************
198* Structures and Typedefs *
199*********************************************************************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG fEFlags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit code qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u7Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringDoublePF;
279 /** Whether the VM-exit was caused by a page-fault during delivery of an
280 * external interrupt or NMI. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns Strict VBox status code (i.e. informational status codes too).
323 * @param pVCpu The cross context virtual CPU structure.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337/**
338 * VMX VM-exit handler, non-strict status code.
339 *
340 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341 *
342 * @returns VBox status code, no informational status code returned.
343 * @param pVCpu The cross context virtual CPU structure.
344 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
345 * out-of-sync. Make sure to update the required
346 * fields before using them.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
364static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
367 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
368 bool fStepping, uint32_t *puIntState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381
382/** @name VM-exit handlers.
383 * @{
384 */
385static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
386static FNVMXEXITHANDLER hmR0VmxExitExtInt;
387static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
394static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
395static FNVMXEXITHANDLER hmR0VmxExitCpuid;
396static FNVMXEXITHANDLER hmR0VmxExitGetsec;
397static FNVMXEXITHANDLER hmR0VmxExitHlt;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
399static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
400static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
401static FNVMXEXITHANDLER hmR0VmxExitVmcall;
402static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
405static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
406static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
407static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
408static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
409static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
413static FNVMXEXITHANDLER hmR0VmxExitMwait;
414static FNVMXEXITHANDLER hmR0VmxExitMtf;
415static FNVMXEXITHANDLER hmR0VmxExitMonitor;
416static FNVMXEXITHANDLER hmR0VmxExitPause;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
419static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
420static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430/** @} */
431
432static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
440static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443/*********************************************************************************************************************************
444* Global Variables *
445*********************************************************************************************************************************/
446#ifdef HMVMX_USE_FUNCTION_TABLE
447
448/**
449 * VMX_EXIT dispatch table.
450 */
451static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452{
453 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493 /* 40 UNDEFINED */ hmR0VmxExitPause,
494 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
510 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
515 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518};
519#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521#ifdef VBOX_STRICT
522static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523{
524 /* 0 */ "(Not Used)",
525 /* 1 */ "VMCALL executed in VMX root operation.",
526 /* 2 */ "VMCLEAR with invalid physical address.",
527 /* 3 */ "VMCLEAR with VMXON pointer.",
528 /* 4 */ "VMLAUNCH with non-clear VMCS.",
529 /* 5 */ "VMRESUME with non-launched VMCS.",
530 /* 6 */ "VMRESUME after VMXOFF",
531 /* 7 */ "VM-entry with invalid control fields.",
532 /* 8 */ "VM-entry with invalid host state fields.",
533 /* 9 */ "VMPTRLD with invalid physical address.",
534 /* 10 */ "VMPTRLD with VMXON pointer.",
535 /* 11 */ "VMPTRLD with incorrect revision identifier.",
536 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537 /* 13 */ "VMWRITE to read-only VMCS component.",
538 /* 14 */ "(Not Used)",
539 /* 15 */ "VMXON executed in VMX root operation.",
540 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541 /* 17 */ "VM-entry with non-launched executing VMCS.",
542 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543 /* 19 */ "VMCALL with non-clear VMCS.",
544 /* 20 */ "VMCALL with invalid VM-exit control fields.",
545 /* 21 */ "(Not Used)",
546 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548 /* 24 */ "VMCALL with invalid SMM-monitor features.",
549 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550 /* 26 */ "VM-entry with events blocked by MOV SS.",
551 /* 27 */ "(Not Used)",
552 /* 28 */ "Invalid operand to INVEPT/INVVPID."
553};
554#endif /* VBOX_STRICT */
555
556
557
558/**
559 * Updates the VM's last error record.
560 *
561 * If there was a VMX instruction error, reads the error data from the VMCS and
562 * updates VCPU's last error record as well.
563 *
564 * @param pVM The cross context VM structure.
565 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
566 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567 * VERR_VMX_INVALID_VMCS_FIELD.
568 * @param rc The error code.
569 */
570static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571{
572 AssertPtr(pVM);
573 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574 || rc == VERR_VMX_UNABLE_TO_START_VM)
575 {
576 AssertPtrReturnVoid(pVCpu);
577 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578 }
579 pVM->hm.s.lLastError = rc;
580}
581
582
583/**
584 * Reads the VM-entry interruption-information field from the VMCS into the VMX
585 * transient structure.
586 *
587 * @returns VBox status code.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 *
590 * @remarks No-long-jump zone!!!
591 */
592DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593{
594 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595 AssertRCReturn(rc, rc);
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Reads the VM-entry exception error code field from the VMCS into
602 * the VMX transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit interruption-information field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 */
641DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642{
643 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644 {
645 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646 AssertRCReturn(rc, rc);
647 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648 }
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-exit interruption error code from the VMCS into the VMX
655 * transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 */
660DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661{
662 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663 {
664 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665 AssertRCReturn(rc, rc);
666 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667 }
668 return VINF_SUCCESS;
669}
670
671
672/**
673 * Reads the VM-exit instruction length field from the VMCS into the VMX
674 * transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVmxTransient Pointer to the VMX transient structure.
678 */
679DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680{
681 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682 {
683 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684 AssertRCReturn(rc, rc);
685 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686 }
687 return VINF_SUCCESS;
688}
689
690
691/**
692 * Reads the VM-exit instruction-information field from the VMCS into
693 * the VMX transient structure.
694 *
695 * @returns VBox status code.
696 * @param pVmxTransient Pointer to the VMX transient structure.
697 */
698DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699{
700 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701 {
702 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703 AssertRCReturn(rc, rc);
704 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705 }
706 return VINF_SUCCESS;
707}
708
709
710/**
711 * Reads the exit code qualification from the VMCS into the VMX transient
712 * structure.
713 *
714 * @returns VBox status code.
715 * @param pVCpu The cross context virtual CPU structure of the
716 * calling EMT. (Required for the VMCS cache case.)
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722 {
723 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the IDT-vectoring information field from the VMCS into the VMX
733 * transient structure.
734 *
735 * @returns VBox status code.
736 * @param pVmxTransient Pointer to the VMX transient structure.
737 *
738 * @remarks No-long-jump zone!!!
739 */
740DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743 {
744 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring error code from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 */
759DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760{
761 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762 {
763 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764 AssertRCReturn(rc, rc);
765 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766 }
767 return VINF_SUCCESS;
768}
769
770
771/**
772 * Enters VMX root mode operation on the current CPU.
773 *
774 * @returns VBox status code.
775 * @param pVM The cross context VM structure. Can be
776 * NULL, after a resume.
777 * @param HCPhysCpuPage Physical address of the VMXON region.
778 * @param pvCpuPage Pointer to the VMXON region.
779 */
780static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781{
782 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784 Assert(pvCpuPage);
785 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787 if (pVM)
788 {
789 /* Write the VMCS revision dword to the VMXON region. */
790 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791 }
792
793 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794 RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796 /* Enable the VMX bit in CR4 if necessary. */
797 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
798
799 /* Enter VMX root mode. */
800 int rc = VMXEnable(HCPhysCpuPage);
801 if (RT_FAILURE(rc))
802 {
803 if (!(uOldCr4 & X86_CR4_VMXE))
804 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806 if (pVM)
807 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808 }
809
810 /* Restore interrupts. */
811 ASMSetFlags(fEFlags);
812 return rc;
813}
814
815
816/**
817 * Exits VMX root mode operation on the current CPU.
818 *
819 * @returns VBox status code.
820 */
821static int hmR0VmxLeaveRootMode(void)
822{
823 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826 RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828 /* If we're for some reason not in VMX root mode, then don't leave it. */
829 RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831 int rc;
832 if (uHostCR4 & X86_CR4_VMXE)
833 {
834 /* Exit VMX root mode and clear the VMX bit in CR4. */
835 VMXDisable();
836 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837 rc = VINF_SUCCESS;
838 }
839 else
840 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842 /* Restore interrupts. */
843 ASMSetFlags(fEFlags);
844 return rc;
845}
846
847
848/**
849 * Allocates and maps one physically contiguous page. The allocated page is
850 * zero'd out. (Used by various VT-x structures).
851 *
852 * @returns IPRT status code.
853 * @param pMemObj Pointer to the ring-0 memory object.
854 * @param ppVirt Where to store the virtual address of the
855 * allocation.
856 * @param pHCPhys Where to store the physical address of the
857 * allocation.
858 */
859DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860{
861 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866 if (RT_FAILURE(rc))
867 return rc;
868 *ppVirt = RTR0MemObjAddress(*pMemObj);
869 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
870 ASMMemZero32(*ppVirt, PAGE_SIZE);
871 return VINF_SUCCESS;
872}
873
874
875/**
876 * Frees and unmaps an allocated physical page.
877 *
878 * @param pMemObj Pointer to the ring-0 memory object.
879 * @param ppVirt Where to re-initialize the virtual address of
880 * allocation as 0.
881 * @param pHCPhys Where to re-initialize the physical address of the
882 * allocation as 0.
883 */
884DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885{
886 AssertPtr(pMemObj);
887 AssertPtr(ppVirt);
888 AssertPtr(pHCPhys);
889 if (*pMemObj != NIL_RTR0MEMOBJ)
890 {
891 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
892 AssertRC(rc);
893 *pMemObj = NIL_RTR0MEMOBJ;
894 *ppVirt = 0;
895 *pHCPhys = 0;
896 }
897}
898
899
900/**
901 * Worker function to free VT-x related structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM The cross context VM structure.
905 */
906static void hmR0VmxStructsFree(PVM pVM)
907{
908 for (VMCPUID i = 0; i < pVM->cCpus; i++)
909 {
910 PVMCPU pVCpu = &pVM->aCpus[i];
911 AssertPtr(pVCpu);
912
913 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921 }
922
923 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926#endif
927}
928
929
930/**
931 * Worker function to allocate VT-x related VM structures.
932 *
933 * @returns IPRT status code.
934 * @param pVM The cross context VM structure.
935 */
936static int hmR0VmxStructsAlloc(PVM pVM)
937{
938 /*
939 * Initialize members up-front so we can cleanup properly on allocation failure.
940 */
941#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944 pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953#endif
954 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#endif
1516 return false;
1517}
1518
1519
1520/**
1521 * Saves a set of guest MSRs back into the guest-CPU context.
1522 *
1523 * @param pVCpu The cross context virtual CPU structure.
1524 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1525 * out-of-sync. Make sure to update the required fields
1526 * before using them.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1536 {
1537 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1538#if HC_ARCH_BITS == 64
1539 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1540 {
1541 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1542 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1543 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1544 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1545 }
1546#endif
1547 }
1548}
1549
1550
1551/**
1552 * Loads a set of guests MSRs to allow read/passthru to the guest.
1553 *
1554 * The name of this function is slightly confusing. This function does NOT
1555 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1556 * common prefix for functions dealing with "lazy restoration" of the shared
1557 * MSRs.
1558 *
1559 * @param pVCpu The cross context virtual CPU structure.
1560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1561 * out-of-sync. Make sure to update the required fields
1562 * before using them.
1563 *
1564 * @remarks No-long-jump zone!!!
1565 */
1566static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1567{
1568 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1569 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1570
1571#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1572 do { \
1573 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1574 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1575 else \
1576 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1577 } while (0)
1578
1579 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1580 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1581 {
1582#if HC_ARCH_BITS == 64
1583 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1584 {
1585 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1586 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1587 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1588 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1589 }
1590#endif
1591 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1592 }
1593
1594#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1595}
1596
1597
1598/**
1599 * Performs lazy restoration of the set of host MSRs if they were previously
1600 * loaded with guest MSR values.
1601 *
1602 * @param pVCpu The cross context virtual CPU structure.
1603 *
1604 * @remarks No-long-jump zone!!!
1605 * @remarks The guest MSRs should have been saved back into the guest-CPU
1606 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1607 */
1608static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1609{
1610 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1611 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1612
1613 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1614 {
1615 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1616#if HC_ARCH_BITS == 64
1617 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1618 {
1619 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1620 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1621 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1622 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1623 }
1624#endif
1625 }
1626 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1627}
1628
1629
1630/**
1631 * Verifies that our cached values of the VMCS controls are all
1632 * consistent with what's actually present in the VMCS.
1633 *
1634 * @returns VBox status code.
1635 * @param pVCpu The cross context virtual CPU structure.
1636 */
1637static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1638{
1639 uint32_t u32Val;
1640 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1643 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1648 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1653 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1654
1655 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1656 AssertRCReturn(rc, rc);
1657 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1658 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1659
1660 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661 {
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1663 AssertRCReturn(rc, rc);
1664 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1665 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1666 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1667 }
1668
1669 return VINF_SUCCESS;
1670}
1671
1672
1673#ifdef VBOX_STRICT
1674/**
1675 * Verifies that our cached host EFER value has not changed
1676 * since we cached it.
1677 *
1678 * @param pVCpu The cross context virtual CPU structure.
1679 */
1680static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1681{
1682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1683
1684 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1685 {
1686 uint64_t u64Val;
1687 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1688 AssertRC(rc);
1689
1690 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1691 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1692 }
1693}
1694
1695
1696/**
1697 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1698 * VMCS are correct.
1699 *
1700 * @param pVCpu The cross context virtual CPU structure.
1701 */
1702static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1703{
1704 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1705
1706 /* Verify MSR counts in the VMCS are what we think it should be. */
1707 uint32_t cMsrs;
1708 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1718 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1719 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1720 {
1721 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1722 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1723 pGuestMsr->u32Msr, cMsrs));
1724
1725 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1726 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1727 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1728
1729 /* Verify that the permissions are as expected in the MSR bitmap. */
1730 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1731 {
1732 VMXMSREXITREAD enmRead;
1733 VMXMSREXITWRITE enmWrite;
1734 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1735 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1736 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1739 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1740 }
1741 else
1742 {
1743 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1744 pGuestMsr->u32Msr, cMsrs));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1746 pGuestMsr->u32Msr, cMsrs));
1747 }
1748 }
1749 }
1750}
1751#endif /* VBOX_STRICT */
1752
1753
1754/**
1755 * Flushes the TLB using EPT.
1756 *
1757 * @returns VBox status code.
1758 * @param pVCpu The cross context virtual CPU structure of the calling
1759 * EMT. Can be NULL depending on @a enmFlush.
1760 * @param enmFlush Type of flush.
1761 *
1762 * @remarks Caller is responsible for making sure this function is called only
1763 * when NestedPaging is supported and providing @a enmFlush that is
1764 * supported by the CPU.
1765 * @remarks Can be called with interrupts disabled.
1766 */
1767static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1768{
1769 uint64_t au64Descriptor[2];
1770 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1771 au64Descriptor[0] = 0;
1772 else
1773 {
1774 Assert(pVCpu);
1775 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1776 }
1777 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1778
1779 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1780 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1781 rc));
1782 if ( RT_SUCCESS(rc)
1783 && pVCpu)
1784 {
1785 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1786 }
1787}
1788
1789
1790/**
1791 * Flushes the TLB using VPID.
1792 *
1793 * @returns VBox status code.
1794 * @param pVM The cross context VM structure.
1795 * @param pVCpu The cross context virtual CPU structure of the calling
1796 * EMT. Can be NULL depending on @a enmFlush.
1797 * @param enmFlush Type of flush.
1798 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1799 * on @a enmFlush).
1800 *
1801 * @remarks Can be called with interrupts disabled.
1802 */
1803static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1804{
1805 NOREF(pVM);
1806 AssertPtr(pVM);
1807 Assert(pVM->hm.s.vmx.fVpid);
1808
1809 uint64_t au64Descriptor[2];
1810 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1811 {
1812 au64Descriptor[0] = 0;
1813 au64Descriptor[1] = 0;
1814 }
1815 else
1816 {
1817 AssertPtr(pVCpu);
1818 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1819 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1820 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1821 au64Descriptor[1] = GCPtr;
1822 }
1823
1824 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1825 AssertMsg(rc == VINF_SUCCESS,
1826 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1827 if ( RT_SUCCESS(rc)
1828 && pVCpu)
1829 {
1830 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1831 }
1832}
1833
1834
1835/**
1836 * Invalidates a guest page by guest virtual address. Only relevant for
1837 * EPT/VPID, otherwise there is nothing really to invalidate.
1838 *
1839 * @returns VBox status code.
1840 * @param pVM The cross context VM structure.
1841 * @param pVCpu The cross context virtual CPU structure.
1842 * @param GCVirt Guest virtual address of the page to invalidate.
1843 */
1844VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1845{
1846 AssertPtr(pVM);
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1855 * See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1858 * function maybe called in a loop with individual addresses.
1859 */
1860 if (pVM->hm.s.vmx.fVpid)
1861 {
1862 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1863 {
1864 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1880 * otherwise there is nothing really to invalidate.
1881 *
1882 * @returns VBox status code.
1883 * @param pVM The cross context VM structure.
1884 * @param pVCpu The cross context virtual CPU structure.
1885 * @param GCPhys Guest physical address of the page to invalidate.
1886 */
1887VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1888{
1889 NOREF(pVM); NOREF(GCPhys);
1890 LogFlowFunc(("%RGp\n", GCPhys));
1891
1892 /*
1893 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1894 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1895 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1896 */
1897 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1898 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1899 return VINF_SUCCESS;
1900}
1901
1902
1903/**
1904 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1905 * case where neither EPT nor VPID is supported by the CPU.
1906 *
1907 * @param pVM The cross context VM structure.
1908 * @param pVCpu The cross context virtual CPU structure.
1909 * @param pCpu Pointer to the global HM struct.
1910 *
1911 * @remarks Called with interrupts disabled.
1912 */
1913static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1914{
1915 AssertPtr(pVCpu);
1916 AssertPtr(pCpu);
1917 NOREF(pVM);
1918
1919 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921 Assert(pCpu->idCpu != NIL_RTCPUID);
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924 pVCpu->hm.s.fForceTLBFlush = false;
1925 return;
1926}
1927
1928
1929/**
1930 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931 *
1932 * @param pVM The cross context VM structure.
1933 * @param pVCpu The cross context virtual CPU structure.
1934 * @param pCpu Pointer to the global HM CPU struct.
1935 * @remarks All references to "ASID" in this function pertains to "VPID" in
1936 * Intel's nomenclature. The reason is, to avoid confusion in compare
1937 * statements since the host-CPU copies are named "ASID".
1938 *
1939 * @remarks Called with interrupts disabled.
1940 */
1941static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942{
1943#ifdef VBOX_WITH_STATISTICS
1944 bool fTlbFlushed = false;
1945# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947 if (!fTlbFlushed) \
1948 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949 } while (0)
1950#else
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953#endif
1954
1955 AssertPtr(pVM);
1956 AssertPtr(pCpu);
1957 AssertPtr(pVCpu);
1958 Assert(pCpu->idCpu != NIL_RTCPUID);
1959
1960 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964 /*
1965 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1966 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1967 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1968 */
1969 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1970 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1971 {
1972 ++pCpu->uCurrentAsid;
1973 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1974 {
1975 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1976 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1977 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1978 }
1979
1980 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1981 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1982 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1983
1984 /*
1985 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1986 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1987 */
1988 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1989 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1990 HMVMX_SET_TAGGED_TLB_FLUSHED();
1991 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1992 }
1993
1994 /* Check for explicit TLB flushes. */
1995 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1996 {
1997 /*
1998 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1999 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2000 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2001 * but not guest-physical mappings.
2002 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2003 */
2004 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2005 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2006 HMVMX_SET_TAGGED_TLB_FLUSHED();
2007 }
2008
2009 pVCpu->hm.s.fForceTLBFlush = false;
2010 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2011
2012 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2013 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2014 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2015 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2016 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2017 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2018 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2019 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2020 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2021
2022 /* Update VMCS with the VPID. */
2023 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2024 AssertRC(rc);
2025
2026#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2027}
2028
2029
2030/**
2031 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2032 *
2033 * @returns VBox status code.
2034 * @param pVM The cross context VM structure.
2035 * @param pVCpu The cross context virtual CPU structure.
2036 * @param pCpu Pointer to the global HM CPU struct.
2037 *
2038 * @remarks Called with interrupts disabled.
2039 */
2040static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2041{
2042 AssertPtr(pVM);
2043 AssertPtr(pVCpu);
2044 AssertPtr(pCpu);
2045 Assert(pCpu->idCpu != NIL_RTCPUID);
2046 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2047 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2048
2049 /*
2050 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2051 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2052 */
2053 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2054 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2055 {
2056 pVCpu->hm.s.fForceTLBFlush = true;
2057 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2058 }
2059
2060 /* Check for explicit TLB flushes. */
2061 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2062 {
2063 pVCpu->hm.s.fForceTLBFlush = true;
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2065 }
2066
2067 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2068 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2069
2070 if (pVCpu->hm.s.fForceTLBFlush)
2071 {
2072 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2073 pVCpu->hm.s.fForceTLBFlush = false;
2074 }
2075}
2076
2077
2078/**
2079 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080 *
2081 * @returns VBox status code.
2082 * @param pVM The cross context VM structure.
2083 * @param pVCpu The cross context virtual CPU structure.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVM);
2091 AssertPtr(pVCpu);
2092 AssertPtr(pCpu);
2093 Assert(pCpu->idCpu != NIL_RTCPUID);
2094 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097 /*
2098 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116 */
2117 pVCpu->hm.s.fForceTLBFlush = true;
2118 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119 }
2120
2121 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 ++pCpu->uCurrentAsid;
2125 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126 {
2127 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130 }
2131
2132 pVCpu->hm.s.fForceTLBFlush = false;
2133 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135 if (pCpu->fFlushAsidBeforeUse)
2136 {
2137 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140 {
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142 pCpu->fFlushAsidBeforeUse = false;
2143 }
2144 else
2145 {
2146 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148 }
2149 }
2150 }
2151
2152 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2153 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2154 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2155 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2156 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2157 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2158 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2159
2160 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2161 AssertRC(rc);
2162}
2163
2164
2165/**
2166 * Flushes the guest TLB entry based on CPU capabilities.
2167 *
2168 * @param pVCpu The cross context virtual CPU structure.
2169 * @param pCpu Pointer to the global HM CPU struct.
2170 */
2171DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2172{
2173#ifdef HMVMX_ALWAYS_FLUSH_TLB
2174 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2175#endif
2176 PVM pVM = pVCpu->CTX_SUFF(pVM);
2177 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2178 {
2179 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2180 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2181 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2182 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2183 default:
2184 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2185 break;
2186 }
2187
2188 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2189}
2190
2191
2192/**
2193 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2194 * TLB entries from the host TLB before VM-entry.
2195 *
2196 * @returns VBox status code.
2197 * @param pVM The cross context VM structure.
2198 */
2199static int hmR0VmxSetupTaggedTlb(PVM pVM)
2200{
2201 /*
2202 * Determine optimal flush type for Nested Paging.
2203 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2204 * guest execution (see hmR3InitFinalizeR0()).
2205 */
2206 if (pVM->hm.s.fNestedPaging)
2207 {
2208 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2209 {
2210 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2212 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2213 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221
2222 /* Make sure the write-back cacheable memory type for EPT is supported. */
2223 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2224 {
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* EPT requires a page-walk length of 4. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237 }
2238 else
2239 {
2240 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246
2247 /*
2248 * Determine optimal flush type for VPID.
2249 */
2250 if (pVM->hm.s.vmx.fVpid)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2253 {
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2255 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2256 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2258 else
2259 {
2260 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2262 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2263 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2264 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2266 pVM->hm.s.vmx.fVpid = false;
2267 }
2268 }
2269 else
2270 {
2271 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2272 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277
2278 /*
2279 * Setup the handler for flushing tagged-TLBs.
2280 */
2281 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2282 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2283 else if (pVM->hm.s.fNestedPaging)
2284 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2285 else if (pVM->hm.s.vmx.fVpid)
2286 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2287 else
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2289 return VINF_SUCCESS;
2290}
2291
2292
2293/**
2294 * Sets up pin-based VM-execution controls in the VMCS.
2295 *
2296 * @returns VBox status code.
2297 * @param pVM The cross context VM structure.
2298 * @param pVCpu The cross context virtual CPU structure.
2299 */
2300static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2301{
2302 AssertPtr(pVM);
2303 AssertPtr(pVCpu);
2304
2305 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2306 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2307
2308 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2309 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2310
2311 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2312 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2313
2314 /* Enable the VMX preemption timer. */
2315 if (pVM->hm.s.vmx.fUsePreemptTimer)
2316 {
2317 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2319 }
2320
2321#ifdef VBOX_WITH_NEW_APIC
2322#if 0
2323 /* Enable posted-interrupt processing. */
2324 if (pVM->hm.s.fPostedIntrs)
2325 {
2326 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329 }
2330#endif
2331#endif
2332
2333 if ((val & zap) != val)
2334 {
2335 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2336 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2337 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2338 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2339 }
2340
2341 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2342 AssertRCReturn(rc, rc);
2343
2344 pVCpu->hm.s.vmx.u32PinCtls = val;
2345 return rc;
2346}
2347
2348
2349/**
2350 * Sets up processor-based VM-execution controls in the VMCS.
2351 *
2352 * @returns VBox status code.
2353 * @param pVM The cross context VM structure.
2354 * @param pVCpu The cross context virtual CPU structure.
2355 */
2356static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2357{
2358 AssertPtr(pVM);
2359 AssertPtr(pVCpu);
2360
2361 int rc = VERR_INTERNAL_ERROR_5;
2362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2364
2365 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2367 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2368 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2369 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2372
2373 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2374 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2375 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2376 {
2377 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2378 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2379 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2380 }
2381
2382 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2383 if (!pVM->hm.s.fNestedPaging)
2384 {
2385 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2386 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2387 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2388 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2389 }
2390
2391 /* Use TPR shadowing if supported by the CPU. */
2392 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2393 {
2394 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2396 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2397 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2398 AssertRCReturn(rc, rc);
2399
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2401 /* CR8 writes cause a VM-exit based on TPR threshold. */
2402 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2403 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2404 }
2405 else
2406 {
2407 /*
2408 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2409 * Set this control only for 64-bit guests.
2410 */
2411 if (pVM->hm.s.fAllow64BitGuests)
2412 {
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2414 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2415 }
2416 }
2417
2418 /* Use MSR-bitmaps if supported by the CPU. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2420 {
2421 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2422
2423 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2424 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2425 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2426 AssertRCReturn(rc, rc);
2427
2428 /*
2429 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2430 * automatically using dedicated fields in the VMCS.
2431 */
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2434 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2435 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437
2438#if HC_ARCH_BITS == 64
2439 /*
2440 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2441 */
2442 if (pVM->hm.s.fAllow64BitGuests)
2443 {
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 }
2449#endif
2450 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2451 }
2452
2453 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2454 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2455 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2456
2457 if ((val & zap) != val)
2458 {
2459 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2460 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2461 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2462 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2463 }
2464
2465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2466 AssertRCReturn(rc, rc);
2467
2468 pVCpu->hm.s.vmx.u32ProcCtls = val;
2469
2470 /*
2471 * Secondary processor-based VM-execution controls.
2472 */
2473 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2474 {
2475 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2476 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2477
2478 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2479 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2480
2481 if (pVM->hm.s.fNestedPaging)
2482 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2483 else
2484 {
2485 /*
2486 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2487 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2488 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2489 */
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2492 }
2493
2494 if (pVM->hm.s.vmx.fVpid)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2496
2497 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2498 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2499
2500#ifdef VBOX_WITH_NEW_APIC
2501#if 0
2502 if (pVM->hm.s.fVirtApicRegs)
2503 {
2504 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2505 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2506
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2509 }
2510#endif
2511#endif
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2529 && pVM->hm.s.vmx.cPleGapTicks
2530 && pVM->hm.s.vmx.cPleWindowTicks)
2531 {
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2535 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2536 AssertRCReturn(rc, rc);
2537 }
2538
2539 if ((val & zap) != val)
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2542 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2548 AssertRCReturn(rc, rc);
2549
2550 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2551 }
2552 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2553 {
2554 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2555 "available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 return VINF_SUCCESS;
2561}
2562
2563
2564/**
2565 * Sets up miscellaneous (everything other than Pin & Processor-based
2566 * VM-execution) control fields in the VMCS.
2567 *
2568 * @returns VBox status code.
2569 * @param pVM The cross context VM structure.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2573{
2574 NOREF(pVM);
2575 AssertPtr(pVM);
2576 AssertPtr(pVCpu);
2577
2578 int rc = VERR_GENERAL_FAILURE;
2579
2580 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2581#if 0
2582 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2585
2586 /*
2587 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2588 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2589 * We thus use the exception bitmap to control it rather than use both.
2590 */
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2592 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2593
2594 /** @todo Explore possibility of using IO-bitmaps. */
2595 /* All IO & IOIO instructions cause VM-exits. */
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2598
2599 /* Initialize the MSR-bitmap area. */
2600 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2601 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2603 AssertRCReturn(rc, rc);
2604#endif
2605
2606 /* Setup MSR auto-load/store area. */
2607 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 AssertRCReturn(rc, rc);
2617
2618 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2620 AssertRCReturn(rc, rc);
2621
2622 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2623#if 0
2624 /* Setup debug controls */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2626 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2627 AssertRCReturn(rc, rc);
2628#endif
2629
2630 return rc;
2631}
2632
2633
2634/**
2635 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2636 *
2637 * @returns VBox status code.
2638 * @param pVM The cross context VM structure.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2642{
2643 AssertPtr(pVM);
2644 AssertPtr(pVCpu);
2645
2646 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2647
2648 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2649
2650 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2652
2653 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2654 and writes, and because recursive #DBs can cause the CPU hang, we must always
2655 intercept #DB. */
2656 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2657
2658 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2659 if (!pVM->hm.s.fNestedPaging)
2660 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2661
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2664 AssertRCReturn(rc, rc);
2665 return rc;
2666}
2667
2668
2669/**
2670 * Sets up the initial guest-state mask. The guest-state mask is consulted
2671 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2672 * for the nested virtualization case (as it would cause a VM-exit).
2673 *
2674 * @param pVCpu The cross context virtual CPU structure.
2675 */
2676static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2677{
2678 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2679 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2740 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2741 */
2742 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744 || !pVM->hm.s.vmx.pRealModeTSS))
2745 {
2746 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2747 return VERR_INTERNAL_ERROR;
2748 }
2749
2750 /* Initialize these always, see hmR3InitFinalizeR0().*/
2751 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2752 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2753
2754 /* Setup the tagged-TLB flush handlers. */
2755 int rc = hmR0VmxSetupTaggedTlb(pVM);
2756 if (RT_FAILURE(rc))
2757 {
2758 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759 return rc;
2760 }
2761
2762 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764#if HC_ARCH_BITS == 64
2765 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2768 {
2769 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770 }
2771#endif
2772
2773 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774 RTCCUINTREG uHostCR4 = ASMGetCR4();
2775 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779 {
2780 PVMCPU pVCpu = &pVM->aCpus[i];
2781 AssertPtr(pVCpu);
2782 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2785 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2788 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2789 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2790
2791 /* Set revision dword at the beginning of the VMCS structure. */
2792 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2793
2794 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2795 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 /* Load this VMCS as the current VMCS. */
2800 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824#if HC_ARCH_BITS == 32
2825 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828#endif
2829
2830 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2831 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2838 }
2839
2840 return VINF_SUCCESS;
2841}
2842
2843
2844/**
2845 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846 * the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVM The cross context VM structure.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2853{
2854 NOREF(pVM); NOREF(pVCpu);
2855
2856 RTCCUINTREG uReg = ASMGetCR0();
2857 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2858 AssertRCReturn(rc, rc);
2859
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 AssertRCReturn(rc, rc);
2863
2864 uReg = ASMGetCR4();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2866 AssertRCReturn(rc, rc);
2867 return rc;
2868}
2869
2870
2871#if HC_ARCH_BITS == 64
2872/**
2873 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2874 * requirements. See hmR0VmxSaveHostSegmentRegs().
2875 */
2876# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2877 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2878 { \
2879 bool fValidSelector = true; \
2880 if ((selValue) & X86_SEL_LDT) \
2881 { \
2882 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2883 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2884 } \
2885 if (fValidSelector) \
2886 { \
2887 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2888 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2889 } \
2890 (selValue) = 0; \
2891 }
2892#endif
2893
2894
2895/**
2896 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2897 * the host-state area in the VMCS.
2898 *
2899 * @returns VBox status code.
2900 * @param pVM The cross context VM structure.
2901 * @param pVCpu The cross context virtual CPU structure.
2902 */
2903DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2904{
2905 int rc = VERR_INTERNAL_ERROR_5;
2906
2907#if HC_ARCH_BITS == 64
2908 /*
2909 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2910 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2911 *
2912 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2913 * Was observed booting Solaris10u10 32-bit guest.
2914 */
2915 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2916 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2917 {
2918 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2919 pVCpu->idCpu));
2920 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2921 }
2922 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2923#endif
2924
2925 /*
2926 * Host DS, ES, FS and GS segment registers.
2927 */
2928#if HC_ARCH_BITS == 64
2929 RTSEL uSelDS = ASMGetDS();
2930 RTSEL uSelES = ASMGetES();
2931 RTSEL uSelFS = ASMGetFS();
2932 RTSEL uSelGS = ASMGetGS();
2933#else
2934 RTSEL uSelDS = 0;
2935 RTSEL uSelES = 0;
2936 RTSEL uSelFS = 0;
2937 RTSEL uSelGS = 0;
2938#endif
2939
2940 /*
2941 * Host CS and SS segment registers.
2942 */
2943 RTSEL uSelCS = ASMGetCS();
2944 RTSEL uSelSS = ASMGetSS();
2945
2946 /*
2947 * Host TR segment register.
2948 */
2949 RTSEL uSelTR = ASMGetTR();
2950
2951#if HC_ARCH_BITS == 64
2952 /*
2953 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2954 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2955 */
2956 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2957 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2958 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2959 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2960# undef VMXLOCAL_ADJUST_HOST_SEG
2961#endif
2962
2963 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2964 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2965 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2966 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2967 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2968 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2969 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2970 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2971 Assert(uSelCS);
2972 Assert(uSelTR);
2973
2974 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2975#if 0
2976 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2977 Assert(uSelSS != 0);
2978#endif
2979
2980 /* Write these host selector fields into the host-state area in the VMCS. */
2981 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2983#if HC_ARCH_BITS == 64
2984 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2985 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2986 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2987 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2988#else
2989 NOREF(uSelDS);
2990 NOREF(uSelES);
2991 NOREF(uSelFS);
2992 NOREF(uSelGS);
2993#endif
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2995 AssertRCReturn(rc, rc);
2996
2997 /*
2998 * Host GDTR and IDTR.
2999 */
3000 RTGDTR Gdtr;
3001 RTIDTR Idtr;
3002 RT_ZERO(Gdtr);
3003 RT_ZERO(Idtr);
3004 ASMGetGDTR(&Gdtr);
3005 ASMGetIDTR(&Idtr);
3006 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3007 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3008 AssertRCReturn(rc, rc);
3009
3010#if HC_ARCH_BITS == 64
3011 /*
3012 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3013 * maximum limit (0xffff) on every VM-exit.
3014 */
3015 if (Gdtr.cbGdt != 0xffff)
3016 {
3017 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3018 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3019 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3020 }
3021
3022 /*
3023 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3024 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3025 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3026 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3027 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3028 * hosts where we are pretty sure it won't cause trouble.
3029 */
3030# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3031 if (Idtr.cbIdt < 0x0fff)
3032# else
3033 if (Idtr.cbIdt != 0xffff)
3034# endif
3035 {
3036 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3037 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3038 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3039 }
3040#endif
3041
3042 /*
3043 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3044 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3045 */
3046 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3047 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3048 VERR_VMX_INVALID_HOST_STATE);
3049
3050 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3051#if HC_ARCH_BITS == 64
3052 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3053
3054 /*
3055 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3056 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3057 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3058 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3059 *
3060 * [1] See Intel spec. 3.5 "System Descriptor Types".
3061 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3062 */
3063 Assert(pDesc->System.u4Type == 11);
3064 if ( pDesc->System.u16LimitLow != 0x67
3065 || pDesc->System.u4LimitHigh)
3066 {
3067 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3068 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3069 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3070 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3071 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3072
3073 /* Store the GDTR here as we need it while restoring TR. */
3074 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3075 }
3076#else
3077 NOREF(pVM);
3078 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079#endif
3080 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host FS base and GS base.
3085 */
3086#if HC_ARCH_BITS == 64
3087 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098#endif
3099 return rc;
3100}
3101
3102
3103/**
3104 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3105 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3106 * the host after every successful VM-exit.
3107 *
3108 * @returns VBox status code.
3109 * @param pVM The cross context VM structure.
3110 * @param pVCpu The cross context virtual CPU structure.
3111 *
3112 * @remarks No-long-jump zone!!!
3113 */
3114DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3115{
3116 NOREF(pVM);
3117
3118 AssertPtr(pVCpu);
3119 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3120
3121 /*
3122 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3123 * rather than swapping them on every VM-entry.
3124 */
3125 hmR0VmxLazySaveHostMsrs(pVCpu);
3126
3127 /*
3128 * Host Sysenter MSRs.
3129 */
3130 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3131#if HC_ARCH_BITS == 32
3132 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3133 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3134#else
3135 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3136 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3137#endif
3138 AssertRCReturn(rc, rc);
3139
3140 /*
3141 * Host EFER MSR.
3142 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3143 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3144 */
3145 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3146 {
3147 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3148 AssertRCReturn(rc, rc);
3149 }
3150
3151 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3152 * hmR0VmxLoadGuestExitCtls() !! */
3153
3154 return rc;
3155}
3156
3157
3158/**
3159 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3160 *
3161 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3162 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3163 * hmR0VMxLoadGuestEntryCtls().
3164 *
3165 * @returns true if we need to load guest EFER, false otherwise.
3166 * @param pVCpu The cross context virtual CPU structure.
3167 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3168 * out-of-sync. Make sure to update the required fields
3169 * before using them.
3170 *
3171 * @remarks Requires EFER, CR4.
3172 * @remarks No-long-jump zone!!!
3173 */
3174static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3175{
3176#ifdef HMVMX_ALWAYS_SWAP_EFER
3177 return true;
3178#endif
3179
3180#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3181 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3182 if (CPUMIsGuestInLongMode(pVCpu))
3183 return false;
3184#endif
3185
3186 PVM pVM = pVCpu->CTX_SUFF(pVM);
3187 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3188 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3189
3190 /*
3191 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3192 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3193 */
3194 if ( CPUMIsGuestInLongMode(pVCpu)
3195 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3196 {
3197 return true;
3198 }
3199
3200 /*
3201 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3202 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3203 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3204 */
3205 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3206 && (pMixedCtx->cr0 & X86_CR0_PG)
3207 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3208 {
3209 /* Assert that host is PAE capable. */
3210 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3211 return true;
3212 }
3213
3214 /** @todo Check the latest Intel spec. for any other bits,
3215 * like SMEP/SMAP? */
3216 return false;
3217}
3218
3219
3220/**
3221 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3222 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3223 * controls".
3224 *
3225 * @returns VBox status code.
3226 * @param pVCpu The cross context virtual CPU structure.
3227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3228 * out-of-sync. Make sure to update the required fields
3229 * before using them.
3230 *
3231 * @remarks Requires EFER.
3232 * @remarks No-long-jump zone!!!
3233 */
3234DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3235{
3236 int rc = VINF_SUCCESS;
3237 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3238 {
3239 PVM pVM = pVCpu->CTX_SUFF(pVM);
3240 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3241 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3242
3243 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3244 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3245
3246 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3247 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3248 {
3249 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3250 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3251 }
3252 else
3253 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3254
3255 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3256 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3257 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3258 {
3259 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3260 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3261 }
3262
3263 /*
3264 * The following should -not- be set (since we're not in SMM mode):
3265 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3266 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3267 */
3268
3269 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3270 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3271
3272 if ((val & zap) != val)
3273 {
3274 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3275 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3276 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3277 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3278 }
3279
3280 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3281 AssertRCReturn(rc, rc);
3282
3283 pVCpu->hm.s.vmx.u32EntryCtls = val;
3284 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3285 }
3286 return rc;
3287}
3288
3289
3290/**
3291 * Sets up the VM-exit controls in the VMCS.
3292 *
3293 * @returns VBox status code.
3294 * @param pVCpu The cross context virtual CPU structure.
3295 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3296 * out-of-sync. Make sure to update the required fields
3297 * before using them.
3298 *
3299 * @remarks Requires EFER.
3300 */
3301DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3302{
3303 NOREF(pMixedCtx);
3304
3305 int rc = VINF_SUCCESS;
3306 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3307 {
3308 PVM pVM = pVCpu->CTX_SUFF(pVM);
3309 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3310 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3311
3312 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3313 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3314
3315 /*
3316 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3317 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3318 */
3319#if HC_ARCH_BITS == 64
3320 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3321 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3322#else
3323 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3324 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3325 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3326 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3327 {
3328 /* The switcher returns to long mode, EFER is managed by the switcher. */
3329 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3330 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3331 }
3332 else
3333 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3334#endif
3335
3336 /* If the newer VMCS fields for managing EFER exists, use it. */
3337 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3338 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3339 {
3340 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3341 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3342 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3343 }
3344
3345 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3346 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3347
3348 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3349 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3350 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3351
3352 if ( pVM->hm.s.vmx.fUsePreemptTimer
3353 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3354 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3355
3356 if ((val & zap) != val)
3357 {
3358 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3359 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3362 }
3363
3364 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3365 AssertRCReturn(rc, rc);
3366
3367 pVCpu->hm.s.vmx.u32ExitCtls = val;
3368 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3369 }
3370 return rc;
3371}
3372
3373
3374/**
3375 * Sets the TPR threshold in the VMCS.
3376 *
3377 * @returns VBox status code.
3378 * @param pVCpu The cross context virtual CPU structure.
3379 * @param u32TprThreshold The TPR threshold (task-priority class only).
3380 */
3381DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3382{
3383 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3384 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
3385 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3386}
3387
3388
3389/**
3390 * Loads the guest APIC and related state.
3391 *
3392 * @returns VBox status code.
3393 * @param pVCpu The cross context virtual CPU structure.
3394 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3395 * out-of-sync. Make sure to update the required fields
3396 * before using them.
3397 */
3398DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3399{
3400 NOREF(pMixedCtx);
3401
3402 int rc = VINF_SUCCESS;
3403 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3404 {
3405 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407 {
3408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410 bool fPendingIntr = false;
3411 uint8_t u8Tpr = 0;
3412 uint8_t u8PendingIntr = 0;
3413 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414 AssertRCReturn(rc, rc);
3415
3416 /*
3417 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3418 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3419 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3420 * the interrupt when we VM-exit for other reasons.
3421 */
3422 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3423 uint32_t u32TprThreshold = 0;
3424 if (fPendingIntr)
3425 {
3426 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3427 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3428 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3429 if (u8PendingPriority <= u8TprPriority)
3430 u32TprThreshold = u8PendingPriority;
3431 else
3432 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3433 }
3434
3435 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3436 AssertRCReturn(rc, rc);
3437 }
3438
3439 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3440 }
3441 return rc;
3442}
3443
3444
3445/**
3446 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3447 *
3448 * @returns Guest's interruptibility-state.
3449 * @param pVCpu The cross context virtual CPU structure.
3450 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3451 * out-of-sync. Make sure to update the required fields
3452 * before using them.
3453 *
3454 * @remarks No-long-jump zone!!!
3455 */
3456DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3457{
3458 /*
3459 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3460 */
3461 uint32_t uIntrState = 0;
3462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3463 {
3464 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3465 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3466 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3467 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 if (pMixedCtx->eflags.Bits.u1IF)
3470 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471 else
3472 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473 }
3474 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475 {
3476 /*
3477 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3478 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3479 */
3480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3481 }
3482 }
3483
3484 /*
3485 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3486 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3487 * setting this would block host-NMIs and IRET will not clear the blocking.
3488 *
3489 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3490 */
3491 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3492 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3493 {
3494 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3495 }
3496
3497 return uIntrState;
3498}
3499
3500
3501/**
3502 * Loads the guest's interruptibility-state into the guest-state area in the
3503 * VMCS.
3504 *
3505 * @returns VBox status code.
3506 * @param pVCpu The cross context virtual CPU structure.
3507 * @param uIntrState The interruptibility-state to set.
3508 */
3509static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3510{
3511 NOREF(pVCpu);
3512 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3513 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3514 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3515 AssertRC(rc);
3516 return rc;
3517}
3518
3519
3520/**
3521 * Loads the exception intercepts required for guest execution in the VMCS.
3522 *
3523 * @returns VBox status code.
3524 * @param pVCpu The cross context virtual CPU structure.
3525 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3526 * out-of-sync. Make sure to update the required fields
3527 * before using them.
3528 */
3529static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3530{
3531 NOREF(pMixedCtx);
3532 int rc = VINF_SUCCESS;
3533 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3534 {
3535 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3536 if (pVCpu->hm.s.fGIMTrapXcptUD)
3537 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3538#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3539 else
3540 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3541#endif
3542
3543 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3544 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3545
3546 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3547 AssertRCReturn(rc, rc);
3548
3549 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3550 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3551 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3552 }
3553 return rc;
3554}
3555
3556
3557/**
3558 * Loads the guest's RIP into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu The cross context virtual CPU structure.
3562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3563 * out-of-sync. Make sure to update the required fields
3564 * before using them.
3565 *
3566 * @remarks No-long-jump zone!!!
3567 */
3568static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3569{
3570 int rc = VINF_SUCCESS;
3571 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3572 {
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3574 AssertRCReturn(rc, rc);
3575
3576 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3577 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3578 HMCPU_CF_VALUE(pVCpu)));
3579 }
3580 return rc;
3581}
3582
3583
3584/**
3585 * Loads the guest's RSP into the guest-state area in the VMCS.
3586 *
3587 * @returns VBox status code.
3588 * @param pVCpu The cross context virtual CPU structure.
3589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3590 * out-of-sync. Make sure to update the required fields
3591 * before using them.
3592 *
3593 * @remarks No-long-jump zone!!!
3594 */
3595static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3596{
3597 int rc = VINF_SUCCESS;
3598 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3599 {
3600 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3601 AssertRCReturn(rc, rc);
3602
3603 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3604 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3605 }
3606 return rc;
3607}
3608
3609
3610/**
3611 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu The cross context virtual CPU structure.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 int rc = VINF_SUCCESS;
3624 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3625 {
3626 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3627 Let us assert it as such and use 32-bit VMWRITE. */
3628 Assert(!(pMixedCtx->rflags.u64 >> 32));
3629 X86EFLAGS Eflags = pMixedCtx->eflags;
3630 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3631 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3632 * These will never be cleared/set, unless some other part of the VMM
3633 * code is buggy - in which case we're better of finding and fixing
3634 * those bugs than hiding them. */
3635 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3636 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3637 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3638 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3639
3640 /*
3641 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3642 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3643 */
3644 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3645 {
3646 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3647 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3648 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3649 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3650 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3651 }
3652
3653 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3654 AssertRCReturn(rc, rc);
3655
3656 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3657 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3658 }
3659 return rc;
3660}
3661
3662
3663/**
3664 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3665 *
3666 * @returns VBox status code.
3667 * @param pVCpu The cross context virtual CPU structure.
3668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3669 * out-of-sync. Make sure to update the required fields
3670 * before using them.
3671 *
3672 * @remarks No-long-jump zone!!!
3673 */
3674DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3675{
3676 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3677 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3678 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3679 AssertRCReturn(rc, rc);
3680 return rc;
3681}
3682
3683
3684/**
3685 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3686 * CR0 is partially shared with the host and we have to consider the FPU bits.
3687 *
3688 * @returns VBox status code.
3689 * @param pVCpu The cross context virtual CPU structure.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775 else
3776 {
3777 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3778 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3779 }
3780 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3781
3782 if (fInterceptNM)
3783 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3784 else
3785 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3786
3787 if (fInterceptMF)
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3789 else
3790 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3791
3792 /* Additional intercepts for debugging, define these yourself explicitly. */
3793#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3794 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3795 | RT_BIT(X86_XCPT_BP)
3796 | RT_BIT(X86_XCPT_DE)
3797 | RT_BIT(X86_XCPT_NM)
3798 | RT_BIT(X86_XCPT_TS)
3799 | RT_BIT(X86_XCPT_UD)
3800 | RT_BIT(X86_XCPT_NP)
3801 | RT_BIT(X86_XCPT_SS)
3802 | RT_BIT(X86_XCPT_GP)
3803 | RT_BIT(X86_XCPT_PF)
3804 | RT_BIT(X86_XCPT_MF)
3805 ;
3806#elif defined(HMVMX_ALWAYS_TRAP_PF)
3807 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3808#endif
3809
3810 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3811
3812 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3813 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3814 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3815 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3816 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3817 else
3818 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3819
3820 u32GuestCR0 |= uSetCR0;
3821 u32GuestCR0 &= uZapCR0;
3822 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3823
3824 /* Write VT-x's view of the guest CR0 into the VMCS. */
3825 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3826 AssertRCReturn(rc, rc);
3827 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3828 uZapCR0));
3829
3830 /*
3831 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3832 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3833 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3834 */
3835 uint32_t u32CR0Mask = 0;
3836 u32CR0Mask = X86_CR0_PE
3837 | X86_CR0_NE
3838 | X86_CR0_WP
3839 | X86_CR0_PG
3840 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3841 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3842 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3843
3844 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3845 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3846 * and @bugref{6944}. */
3847#if 0
3848 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3849 u32CR0Mask &= ~X86_CR0_PE;
3850#endif
3851 if (pVM->hm.s.fNestedPaging)
3852 u32CR0Mask &= ~X86_CR0_WP;
3853
3854 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3855 if (fInterceptNM)
3856 {
3857 u32CR0Mask |= X86_CR0_TS
3858 | X86_CR0_MP;
3859 }
3860
3861 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3862 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3863 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3864 AssertRCReturn(rc, rc);
3865 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3866
3867 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3868 }
3869 return rc;
3870}
3871
3872
3873/**
3874 * Loads the guest control registers (CR3, CR4) into the guest-state area
3875 * in the VMCS.
3876 *
3877 * @returns VBox strict status code.
3878 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3879 * without unrestricted guest access and the VMMDev is not presently
3880 * mapped (e.g. EFI32).
3881 *
3882 * @param pVCpu The cross context virtual CPU structure.
3883 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3884 * out-of-sync. Make sure to update the required fields
3885 * before using them.
3886 *
3887 * @remarks No-long-jump zone!!!
3888 */
3889static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3890{
3891 int rc = VINF_SUCCESS;
3892 PVM pVM = pVCpu->CTX_SUFF(pVM);
3893
3894 /*
3895 * Guest CR2.
3896 * It's always loaded in the assembler code. Nothing to do here.
3897 */
3898
3899 /*
3900 * Guest CR3.
3901 */
3902 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3903 {
3904 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3905 if (pVM->hm.s.fNestedPaging)
3906 {
3907 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3908
3909 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3910 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3911 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3912 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3913
3914 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3915 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3916 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3917
3918 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3919 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3920 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3921 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3922 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3923 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3924 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3925
3926 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3927 AssertRCReturn(rc, rc);
3928 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3929
3930 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3931 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3932 {
3933 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3934 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3935 {
3936 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3937 AssertRCReturn(rc, rc);
3938 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3939 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3940 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3941 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3942 AssertRCReturn(rc, rc);
3943 }
3944
3945 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3946 have Unrestricted Execution to handle the guest when it's not using paging. */
3947 GCPhysGuestCR3 = pMixedCtx->cr3;
3948 }
3949 else
3950 {
3951 /*
3952 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3953 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3954 * EPT takes care of translating it to host-physical addresses.
3955 */
3956 RTGCPHYS GCPhys;
3957 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3958
3959 /* We obtain it here every time as the guest could have relocated this PCI region. */
3960 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3961 if (RT_SUCCESS(rc))
3962 { /* likely */ }
3963 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3964 {
3965 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3966 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3967 }
3968 else
3969 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3970
3971 GCPhysGuestCR3 = GCPhys;
3972 }
3973
3974 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3975 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3976 }
3977 else
3978 {
3979 /* Non-nested paging case, just use the hypervisor's CR3. */
3980 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3981
3982 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3983 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3984 }
3985 AssertRCReturn(rc, rc);
3986
3987 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3988 }
3989
3990 /*
3991 * Guest CR4.
3992 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3993 */
3994 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3995 {
3996 Assert(!(pMixedCtx->cr4 >> 32));
3997 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3998
3999 /* The guest's view of its CR4 is unblemished. */
4000 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4001 AssertRCReturn(rc, rc);
4002 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4003
4004 /* Setup VT-x's view of the guest CR4. */
4005 /*
4006 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4007 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4008 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4009 */
4010 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4011 {
4012 Assert(pVM->hm.s.vmx.pRealModeTSS);
4013 Assert(PDMVmmDevHeapIsEnabled(pVM));
4014 u32GuestCR4 &= ~X86_CR4_VME;
4015 }
4016
4017 if (pVM->hm.s.fNestedPaging)
4018 {
4019 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4020 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4021 {
4022 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4023 u32GuestCR4 |= X86_CR4_PSE;
4024 /* Our identity mapping is a 32-bit page directory. */
4025 u32GuestCR4 &= ~X86_CR4_PAE;
4026 }
4027 /* else use guest CR4.*/
4028 }
4029 else
4030 {
4031 /*
4032 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4033 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4034 */
4035 switch (pVCpu->hm.s.enmShadowMode)
4036 {
4037 case PGMMODE_REAL: /* Real-mode. */
4038 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4039 case PGMMODE_32_BIT: /* 32-bit paging. */
4040 {
4041 u32GuestCR4 &= ~X86_CR4_PAE;
4042 break;
4043 }
4044
4045 case PGMMODE_PAE: /* PAE paging. */
4046 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4047 {
4048 u32GuestCR4 |= X86_CR4_PAE;
4049 break;
4050 }
4051
4052 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4053 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4054#ifdef VBOX_ENABLE_64_BITS_GUESTS
4055 break;
4056#endif
4057 default:
4058 AssertFailed();
4059 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4060 }
4061 }
4062
4063 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4064 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4065 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4066 u32GuestCR4 |= uSetCR4;
4067 u32GuestCR4 &= uZapCR4;
4068
4069 /* Write VT-x's view of the guest CR4 into the VMCS. */
4070 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4071 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4072 AssertRCReturn(rc, rc);
4073
4074 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4075 uint32_t u32CR4Mask = X86_CR4_VME
4076 | X86_CR4_PAE
4077 | X86_CR4_PGE
4078 | X86_CR4_PSE
4079 | X86_CR4_VMXE;
4080 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4081 u32CR4Mask |= X86_CR4_OSXSAVE;
4082 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4083 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4084 AssertRCReturn(rc, rc);
4085
4086 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4087 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4088
4089 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4090 }
4091 return rc;
4092}
4093
4094
4095/**
4096 * Loads the guest debug registers into the guest-state area in the VMCS.
4097 *
4098 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4099 *
4100 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4101 *
4102 * @returns VBox status code.
4103 * @param pVCpu The cross context virtual CPU structure.
4104 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4105 * out-of-sync. Make sure to update the required fields
4106 * before using them.
4107 *
4108 * @remarks No-long-jump zone!!!
4109 */
4110static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4111{
4112 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4113 return VINF_SUCCESS;
4114
4115#ifdef VBOX_STRICT
4116 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4117 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4118 {
4119 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4120 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4121 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4122 }
4123#endif
4124
4125 int rc;
4126 PVM pVM = pVCpu->CTX_SUFF(pVM);
4127 bool fSteppingDB = false;
4128 bool fInterceptMovDRx = false;
4129 if (pVCpu->hm.s.fSingleInstruction)
4130 {
4131 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4132 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4133 {
4134 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4135 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4136 AssertRCReturn(rc, rc);
4137 Assert(fSteppingDB == false);
4138 }
4139 else
4140 {
4141 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4142 pVCpu->hm.s.fClearTrapFlag = true;
4143 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4144 fSteppingDB = true;
4145 }
4146 }
4147
4148 if ( fSteppingDB
4149 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4150 {
4151 /*
4152 * Use the combined guest and host DRx values found in the hypervisor
4153 * register set because the debugger has breakpoints active or someone
4154 * is single stepping on the host side without a monitor trap flag.
4155 *
4156 * Note! DBGF expects a clean DR6 state before executing guest code.
4157 */
4158#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4159 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4160 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4161 {
4162 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4163 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4164 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4165 }
4166 else
4167#endif
4168 if (!CPUMIsHyperDebugStateActive(pVCpu))
4169 {
4170 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4171 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4172 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4173 }
4174
4175 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4176 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4177 AssertRCReturn(rc, rc);
4178
4179 pVCpu->hm.s.fUsingHyperDR7 = true;
4180 fInterceptMovDRx = true;
4181 }
4182 else
4183 {
4184 /*
4185 * If the guest has enabled debug registers, we need to load them prior to
4186 * executing guest code so they'll trigger at the right time.
4187 */
4188 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4189 {
4190#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4191 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4192 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4193 {
4194 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4195 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4196 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4197 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4198 }
4199 else
4200#endif
4201 if (!CPUMIsGuestDebugStateActive(pVCpu))
4202 {
4203 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4204 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4205 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4206 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4207 }
4208 Assert(!fInterceptMovDRx);
4209 }
4210 /*
4211 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4212 * must intercept #DB in order to maintain a correct DR6 guest value, and
4213 * because we need to intercept it to prevent nested #DBs from hanging the
4214 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4215 */
4216#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4217 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4218 && !CPUMIsGuestDebugStateActive(pVCpu))
4219#else
4220 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4221#endif
4222 {
4223 fInterceptMovDRx = true;
4224 }
4225
4226 /* Update guest DR7. */
4227 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4228 AssertRCReturn(rc, rc);
4229
4230 pVCpu->hm.s.fUsingHyperDR7 = false;
4231 }
4232
4233 /*
4234 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4235 */
4236 if (fInterceptMovDRx)
4237 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4238 else
4239 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4241 AssertRCReturn(rc, rc);
4242
4243 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4244 return VINF_SUCCESS;
4245}
4246
4247
4248#ifdef VBOX_STRICT
4249/**
4250 * Strict function to validate segment registers.
4251 *
4252 * @remarks ASSUMES CR0 is up to date.
4253 */
4254static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4255{
4256 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4257 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4258 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4259 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4260 && ( !CPUMIsGuestInRealModeEx(pCtx)
4261 && !CPUMIsGuestInV86ModeEx(pCtx)))
4262 {
4263 /* Protected mode checks */
4264 /* CS */
4265 Assert(pCtx->cs.Attr.n.u1Present);
4266 Assert(!(pCtx->cs.Attr.u & 0xf00));
4267 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4268 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4269 || !(pCtx->cs.Attr.n.u1Granularity));
4270 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4271 || (pCtx->cs.Attr.n.u1Granularity));
4272 /* CS cannot be loaded with NULL in protected mode. */
4273 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4274 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4275 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4276 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4277 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4278 else
4279 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4280 /* SS */
4281 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4282 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4283 if ( !(pCtx->cr0 & X86_CR0_PE)
4284 || pCtx->cs.Attr.n.u4Type == 3)
4285 {
4286 Assert(!pCtx->ss.Attr.n.u2Dpl);
4287 }
4288 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4289 {
4290 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4291 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4292 Assert(pCtx->ss.Attr.n.u1Present);
4293 Assert(!(pCtx->ss.Attr.u & 0xf00));
4294 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4295 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4296 || !(pCtx->ss.Attr.n.u1Granularity));
4297 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4298 || (pCtx->ss.Attr.n.u1Granularity));
4299 }
4300 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4301 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4302 {
4303 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4304 Assert(pCtx->ds.Attr.n.u1Present);
4305 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4306 Assert(!(pCtx->ds.Attr.u & 0xf00));
4307 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4308 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4309 || !(pCtx->ds.Attr.n.u1Granularity));
4310 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4311 || (pCtx->ds.Attr.n.u1Granularity));
4312 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4313 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4314 }
4315 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4316 {
4317 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4318 Assert(pCtx->es.Attr.n.u1Present);
4319 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4320 Assert(!(pCtx->es.Attr.u & 0xf00));
4321 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4322 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4323 || !(pCtx->es.Attr.n.u1Granularity));
4324 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4325 || (pCtx->es.Attr.n.u1Granularity));
4326 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4327 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4328 }
4329 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4330 {
4331 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4332 Assert(pCtx->fs.Attr.n.u1Present);
4333 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4334 Assert(!(pCtx->fs.Attr.u & 0xf00));
4335 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4336 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4337 || !(pCtx->fs.Attr.n.u1Granularity));
4338 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4339 || (pCtx->fs.Attr.n.u1Granularity));
4340 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4341 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4342 }
4343 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4344 {
4345 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4346 Assert(pCtx->gs.Attr.n.u1Present);
4347 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4348 Assert(!(pCtx->gs.Attr.u & 0xf00));
4349 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4350 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4351 || !(pCtx->gs.Attr.n.u1Granularity));
4352 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4353 || (pCtx->gs.Attr.n.u1Granularity));
4354 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4355 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4356 }
4357 /* 64-bit capable CPUs. */
4358# if HC_ARCH_BITS == 64
4359 Assert(!(pCtx->cs.u64Base >> 32));
4360 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4361 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4362 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4363# endif
4364 }
4365 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4366 || ( CPUMIsGuestInRealModeEx(pCtx)
4367 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4368 {
4369 /* Real and v86 mode checks. */
4370 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4371 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4372 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4373 {
4374 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4375 }
4376 else
4377 {
4378 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4379 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4380 }
4381
4382 /* CS */
4383 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4384 Assert(pCtx->cs.u32Limit == 0xffff);
4385 Assert(u32CSAttr == 0xf3);
4386 /* SS */
4387 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4388 Assert(pCtx->ss.u32Limit == 0xffff);
4389 Assert(u32SSAttr == 0xf3);
4390 /* DS */
4391 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4392 Assert(pCtx->ds.u32Limit == 0xffff);
4393 Assert(u32DSAttr == 0xf3);
4394 /* ES */
4395 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4396 Assert(pCtx->es.u32Limit == 0xffff);
4397 Assert(u32ESAttr == 0xf3);
4398 /* FS */
4399 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4400 Assert(pCtx->fs.u32Limit == 0xffff);
4401 Assert(u32FSAttr == 0xf3);
4402 /* GS */
4403 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4404 Assert(pCtx->gs.u32Limit == 0xffff);
4405 Assert(u32GSAttr == 0xf3);
4406 /* 64-bit capable CPUs. */
4407# if HC_ARCH_BITS == 64
4408 Assert(!(pCtx->cs.u64Base >> 32));
4409 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4410 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4411 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4412# endif
4413 }
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Writes a guest segment register into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu The cross context virtual CPU structure.
4423 * @param idxSel Index of the selector in the VMCS.
4424 * @param idxLimit Index of the segment limit in the VMCS.
4425 * @param idxBase Index of the segment base in the VMCS.
4426 * @param idxAccess Index of the access rights of the segment in the VMCS.
4427 * @param pSelReg Pointer to the segment selector.
4428 *
4429 * @remarks No-long-jump zone!!!
4430 */
4431static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4432 uint32_t idxAccess, PCPUMSELREG pSelReg)
4433{
4434 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4435 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4436 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4437 AssertRCReturn(rc, rc);
4438
4439 uint32_t u32Access = pSelReg->Attr.u;
4440 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4441 {
4442 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4443 u32Access = 0xf3;
4444 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4445 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4446 }
4447 else
4448 {
4449 /*
4450 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4451 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4452 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4453 * loaded in protected-mode have their attribute as 0.
4454 */
4455 if (!u32Access)
4456 u32Access = X86DESCATTR_UNUSABLE;
4457 }
4458
4459 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4460 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4461 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4462
4463 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4464 AssertRCReturn(rc, rc);
4465 return rc;
4466}
4467
4468
4469/**
4470 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4471 * into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVCpu The cross context virtual CPU structure.
4475 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4476 * out-of-sync. Make sure to update the required fields
4477 * before using them.
4478 *
4479 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4480 * @remarks No-long-jump zone!!!
4481 */
4482static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4483{
4484 int rc = VERR_INTERNAL_ERROR_5;
4485 PVM pVM = pVCpu->CTX_SUFF(pVM);
4486
4487 /*
4488 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4489 */
4490 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4491 {
4492 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4493 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4494 {
4495 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4496 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4498 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4501 }
4502
4503#ifdef VBOX_WITH_REM
4504 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4505 {
4506 Assert(pVM->hm.s.vmx.pRealModeTSS);
4507 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4508 if ( pVCpu->hm.s.vmx.fWasInRealMode
4509 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4510 {
4511 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4512 in real-mode (e.g. OpenBSD 4.0) */
4513 REMFlushTBs(pVM);
4514 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4515 pVCpu->hm.s.vmx.fWasInRealMode = false;
4516 }
4517 }
4518#endif
4519 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4520 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4521 AssertRCReturn(rc, rc);
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4523 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4526 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4529 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4532 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4535 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4536 AssertRCReturn(rc, rc);
4537
4538#ifdef VBOX_STRICT
4539 /* Validate. */
4540 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4541#endif
4542
4543 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4544 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4545 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4546 }
4547
4548 /*
4549 * Guest TR.
4550 */
4551 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4552 {
4553 /*
4554 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4555 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4556 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4557 */
4558 uint16_t u16Sel = 0;
4559 uint32_t u32Limit = 0;
4560 uint64_t u64Base = 0;
4561 uint32_t u32AccessRights = 0;
4562
4563 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4564 {
4565 u16Sel = pMixedCtx->tr.Sel;
4566 u32Limit = pMixedCtx->tr.u32Limit;
4567 u64Base = pMixedCtx->tr.u64Base;
4568 u32AccessRights = pMixedCtx->tr.Attr.u;
4569 }
4570 else
4571 {
4572 Assert(pVM->hm.s.vmx.pRealModeTSS);
4573 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4574
4575 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4576 RTGCPHYS GCPhys;
4577 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4578 AssertRCReturn(rc, rc);
4579
4580 X86DESCATTR DescAttr;
4581 DescAttr.u = 0;
4582 DescAttr.n.u1Present = 1;
4583 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4584
4585 u16Sel = 0;
4586 u32Limit = HM_VTX_TSS_SIZE;
4587 u64Base = GCPhys; /* in real-mode phys = virt. */
4588 u32AccessRights = DescAttr.u;
4589 }
4590
4591 /* Validate. */
4592 Assert(!(u16Sel & RT_BIT(2)));
4593 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4594 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4595 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4596 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4597 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4598 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4599 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4600 Assert( (u32Limit & 0xfff) == 0xfff
4601 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4602 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4603 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4604
4605 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4606 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4607 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4609 AssertRCReturn(rc, rc);
4610
4611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4612 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4613 }
4614
4615 /*
4616 * Guest GDTR.
4617 */
4618 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4619 {
4620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4621 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4622 AssertRCReturn(rc, rc);
4623
4624 /* Validate. */
4625 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4626
4627 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4628 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4629 }
4630
4631 /*
4632 * Guest LDTR.
4633 */
4634 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4635 {
4636 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4637 uint32_t u32Access = 0;
4638 if (!pMixedCtx->ldtr.Attr.u)
4639 u32Access = X86DESCATTR_UNUSABLE;
4640 else
4641 u32Access = pMixedCtx->ldtr.Attr.u;
4642
4643 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4645 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4647 AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4661 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4672 {
4673 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4674 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4675 AssertRCReturn(rc, rc);
4676
4677 /* Validate. */
4678 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4679
4680 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4681 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4682 }
4683
4684 return VINF_SUCCESS;
4685}
4686
4687
4688/**
4689 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4690 * areas.
4691 *
4692 * These MSRs will automatically be loaded to the host CPU on every successful
4693 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4694 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4695 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4696 *
4697 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4698 *
4699 * @returns VBox status code.
4700 * @param pVCpu The cross context virtual CPU structure.
4701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4702 * out-of-sync. Make sure to update the required fields
4703 * before using them.
4704 *
4705 * @remarks No-long-jump zone!!!
4706 */
4707static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4708{
4709 AssertPtr(pVCpu);
4710 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4711
4712 /*
4713 * MSRs that we use the auto-load/store MSR area in the VMCS.
4714 */
4715 PVM pVM = pVCpu->CTX_SUFF(pVM);
4716 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4717 {
4718 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4719#if HC_ARCH_BITS == 32
4720 if (pVM->hm.s.fAllow64BitGuests)
4721 {
4722 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4723 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4724 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4725 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4726 AssertRCReturn(rc, rc);
4727# ifdef LOG_ENABLED
4728 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4729 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4730 {
4731 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4732 pMsr->u64Value));
4733 }
4734# endif
4735 }
4736#endif
4737 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4738 }
4739
4740 /*
4741 * Guest Sysenter MSRs.
4742 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4743 * VM-exits on WRMSRs for these MSRs.
4744 */
4745 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4746 {
4747 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4748 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4749 }
4750
4751 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4752 {
4753 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4754 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4755 }
4756
4757 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4758 {
4759 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4760 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4761 }
4762
4763 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4764 {
4765 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4766 {
4767 /*
4768 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4769 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4770 */
4771 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4772 {
4773 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4774 AssertRCReturn(rc,rc);
4775 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4776 }
4777 else
4778 {
4779 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4780 NULL /* pfAddedAndUpdated */);
4781 AssertRCReturn(rc, rc);
4782
4783 /* We need to intercept reads too, see @bugref{7386#c16}. */
4784 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4785 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4786 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4787 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4788 }
4789 }
4790 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4791 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4792 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4793 }
4794
4795 return VINF_SUCCESS;
4796}
4797
4798
4799/**
4800 * Loads the guest activity state into the guest-state area in the VMCS.
4801 *
4802 * @returns VBox status code.
4803 * @param pVCpu The cross context virtual CPU structure.
4804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805 * out-of-sync. Make sure to update the required fields
4806 * before using them.
4807 *
4808 * @remarks No-long-jump zone!!!
4809 */
4810static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4811{
4812 NOREF(pMixedCtx);
4813 /** @todo See if we can make use of other states, e.g.
4814 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4815 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4816 {
4817 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4818 AssertRCReturn(rc, rc);
4819
4820 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4821 }
4822 return VINF_SUCCESS;
4823}
4824
4825
4826#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4827/**
4828 * Check if guest state allows safe use of 32-bit switcher again.
4829 *
4830 * Segment bases and protected mode structures must be 32-bit addressable
4831 * because the 32-bit switcher will ignore high dword when writing these VMCS
4832 * fields. See @bugref{8432} for details.
4833 *
4834 * @returns true if safe, false if must continue to use the 64-bit switcher.
4835 * @param pVCpu The cross context virtual CPU structure.
4836 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4837 * out-of-sync. Make sure to update the required fields
4838 * before using them.
4839 *
4840 * @remarks No-long-jump zone!!!
4841 */
4842static bool hmR0VmxIs32BitSwitcherSafe(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4843{
4844 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4845 return false;
4846 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4847 return false;
4848 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4849 return false;
4850 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4851 return false;
4852 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4853 return false;
4854 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4855 return false;
4856 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4857 return false;
4858 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4859 return false;
4860 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4861 return false;
4862 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4863 return false;
4864 /* All good, bases are 32-bit. */
4865 return true;
4866}
4867#endif
4868
4869
4870/**
4871 * Sets up the appropriate function to run guest code.
4872 *
4873 * @returns VBox status code.
4874 * @param pVCpu The cross context virtual CPU structure.
4875 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4876 * out-of-sync. Make sure to update the required fields
4877 * before using them.
4878 *
4879 * @remarks No-long-jump zone!!!
4880 */
4881static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4882{
4883 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4884 {
4885#ifndef VBOX_ENABLE_64_BITS_GUESTS
4886 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4887#endif
4888 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4889#if HC_ARCH_BITS == 32
4890 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4891 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4892 {
4893 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4894 {
4895 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4896 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4897 | HM_CHANGED_VMX_ENTRY_CTLS
4898 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4899 }
4900 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4901
4902 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4903 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4904 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4905 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4906 }
4907#else
4908 /* 64-bit host. */
4909 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4910#endif
4911 }
4912 else
4913 {
4914 /* Guest is not in long mode, use the 32-bit handler. */
4915#if HC_ARCH_BITS == 32
4916 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4917 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4918 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4919 {
4920 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4921 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4922 | HM_CHANGED_VMX_ENTRY_CTLS
4923 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4924 }
4925# ifdef VBOX_ENABLE_64_BITS_GUESTS
4926 /*
4927 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4928 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4929 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4930 * the much faster 32-bit switcher again.
4931 */
4932 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4933 {
4934 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4935 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4936 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4937 }
4938 else
4939 {
4940 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4941 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4942 || hmR0VmxIs32BitSwitcherSafe(pVCpu, pMixedCtx))
4943 {
4944 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4945 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4946 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4947 | HM_CHANGED_VMX_ENTRY_CTLS
4948 | HM_CHANGED_VMX_EXIT_CTLS
4949 | HM_CHANGED_HOST_CONTEXT);
4950 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4951 }
4952 }
4953# else
4954 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4955# endif
4956#else
4957 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4958#endif
4959 }
4960 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4961 return VINF_SUCCESS;
4962}
4963
4964
4965/**
4966 * Wrapper for running the guest code in VT-x.
4967 *
4968 * @returns VBox status code, no informational status codes.
4969 * @param pVM The cross context VM structure.
4970 * @param pVCpu The cross context virtual CPU structure.
4971 * @param pCtx Pointer to the guest-CPU context.
4972 *
4973 * @remarks No-long-jump zone!!!
4974 */
4975DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4976{
4977 /*
4978 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4979 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4980 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4981 */
4982 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4983 /** @todo Add stats for resume vs launch. */
4984#ifdef VBOX_WITH_KERNEL_USING_XMM
4985 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4986#else
4987 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4988#endif
4989 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4990 return rc;
4991}
4992
4993
4994/**
4995 * Reports world-switch error and dumps some useful debug info.
4996 *
4997 * @param pVM The cross context VM structure.
4998 * @param pVCpu The cross context virtual CPU structure.
4999 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5000 * @param pCtx Pointer to the guest-CPU context.
5001 * @param pVmxTransient Pointer to the VMX transient structure (only
5002 * exitReason updated).
5003 */
5004static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5005{
5006 Assert(pVM);
5007 Assert(pVCpu);
5008 Assert(pCtx);
5009 Assert(pVmxTransient);
5010 HMVMX_ASSERT_PREEMPT_SAFE();
5011
5012 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5013 switch (rcVMRun)
5014 {
5015 case VERR_VMX_INVALID_VMXON_PTR:
5016 AssertFailed();
5017 break;
5018 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5019 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5020 {
5021 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5022 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5023 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5024 AssertRC(rc);
5025
5026 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5027 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5028 Cannot do it here as we may have been long preempted. */
5029
5030#ifdef VBOX_STRICT
5031 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5032 pVmxTransient->uExitReason));
5033 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5034 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5035 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5036 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5037 else
5038 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5039 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5040 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5041
5042 /* VMX control bits. */
5043 uint32_t u32Val;
5044 uint64_t u64Val;
5045 RTHCUINTREG uHCReg;
5046 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5049 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5050 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5051 {
5052 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5054 }
5055 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5056 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5057 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5058 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5059 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5060 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5061 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5062 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5063 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5064 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5065 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5066 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5067 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5068 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5069 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5070 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5071 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5072 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5073 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5074 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5075 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5076 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5077 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5078 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5079 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5080 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5081 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5082 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5083 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5084 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5085 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5086 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5087 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5088 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5089 if (pVM->hm.s.fNestedPaging)
5090 {
5091 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5092 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5093 }
5094
5095 /* Guest bits. */
5096 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5097 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5098 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5099 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5100 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5101 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5102 if (pVM->hm.s.vmx.fVpid)
5103 {
5104 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5105 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5106 }
5107
5108 /* Host bits. */
5109 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5110 Log4(("Host CR0 %#RHr\n", uHCReg));
5111 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5112 Log4(("Host CR3 %#RHr\n", uHCReg));
5113 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5114 Log4(("Host CR4 %#RHr\n", uHCReg));
5115
5116 RTGDTR HostGdtr;
5117 PCX86DESCHC pDesc;
5118 ASMGetGDTR(&HostGdtr);
5119 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5120 Log4(("Host CS %#08x\n", u32Val));
5121 if (u32Val < HostGdtr.cbGdt)
5122 {
5123 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5124 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5125 }
5126
5127 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5128 Log4(("Host DS %#08x\n", u32Val));
5129 if (u32Val < HostGdtr.cbGdt)
5130 {
5131 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5132 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5133 }
5134
5135 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5136 Log4(("Host ES %#08x\n", u32Val));
5137 if (u32Val < HostGdtr.cbGdt)
5138 {
5139 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5140 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5141 }
5142
5143 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5144 Log4(("Host FS %#08x\n", u32Val));
5145 if (u32Val < HostGdtr.cbGdt)
5146 {
5147 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5148 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5149 }
5150
5151 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5152 Log4(("Host GS %#08x\n", u32Val));
5153 if (u32Val < HostGdtr.cbGdt)
5154 {
5155 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5156 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5157 }
5158
5159 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5160 Log4(("Host SS %#08x\n", u32Val));
5161 if (u32Val < HostGdtr.cbGdt)
5162 {
5163 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5164 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5165 }
5166
5167 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5168 Log4(("Host TR %#08x\n", u32Val));
5169 if (u32Val < HostGdtr.cbGdt)
5170 {
5171 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5172 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5173 }
5174
5175 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5176 Log4(("Host TR Base %#RHv\n", uHCReg));
5177 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5178 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5179 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5180 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5181 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5182 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5183 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5184 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5185 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5186 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5187 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5188 Log4(("Host RSP %#RHv\n", uHCReg));
5189 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5190 Log4(("Host RIP %#RHv\n", uHCReg));
5191# if HC_ARCH_BITS == 64
5192 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5193 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5194 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5195 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5196 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5197 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5198# endif
5199#endif /* VBOX_STRICT */
5200 break;
5201 }
5202
5203 default:
5204 /* Impossible */
5205 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5206 break;
5207 }
5208 NOREF(pVM); NOREF(pCtx);
5209}
5210
5211
5212#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5213#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5214# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5215#endif
5216#ifdef VBOX_STRICT
5217static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5218{
5219 switch (idxField)
5220 {
5221 case VMX_VMCS_GUEST_RIP:
5222 case VMX_VMCS_GUEST_RSP:
5223 case VMX_VMCS_GUEST_SYSENTER_EIP:
5224 case VMX_VMCS_GUEST_SYSENTER_ESP:
5225 case VMX_VMCS_GUEST_GDTR_BASE:
5226 case VMX_VMCS_GUEST_IDTR_BASE:
5227 case VMX_VMCS_GUEST_CS_BASE:
5228 case VMX_VMCS_GUEST_DS_BASE:
5229 case VMX_VMCS_GUEST_ES_BASE:
5230 case VMX_VMCS_GUEST_FS_BASE:
5231 case VMX_VMCS_GUEST_GS_BASE:
5232 case VMX_VMCS_GUEST_SS_BASE:
5233 case VMX_VMCS_GUEST_LDTR_BASE:
5234 case VMX_VMCS_GUEST_TR_BASE:
5235 case VMX_VMCS_GUEST_CR3:
5236 return true;
5237 }
5238 return false;
5239}
5240
5241static bool hmR0VmxIsValidReadField(uint32_t idxField)
5242{
5243 switch (idxField)
5244 {
5245 /* Read-only fields. */
5246 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5247 return true;
5248 }
5249 /* Remaining readable fields should also be writable. */
5250 return hmR0VmxIsValidWriteField(idxField);
5251}
5252#endif /* VBOX_STRICT */
5253
5254
5255/**
5256 * Executes the specified handler in 64-bit mode.
5257 *
5258 * @returns VBox status code (no informational status codes).
5259 * @param pVM The cross context VM structure.
5260 * @param pVCpu The cross context virtual CPU structure.
5261 * @param pCtx Pointer to the guest CPU context.
5262 * @param enmOp The operation to perform.
5263 * @param cParams Number of parameters.
5264 * @param paParam Array of 32-bit parameters.
5265 */
5266VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5267 uint32_t cParams, uint32_t *paParam)
5268{
5269 NOREF(pCtx);
5270
5271 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5272 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5273 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5274 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5275
5276#ifdef VBOX_STRICT
5277 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5278 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5279
5280 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5281 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5282#endif
5283
5284 /* Disable interrupts. */
5285 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5286
5287#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5288 RTCPUID idHostCpu = RTMpCpuId();
5289 CPUMR0SetLApic(pVCpu, idHostCpu);
5290#endif
5291
5292 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5293 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5294
5295 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5296 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5297 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5298
5299 /* Leave VMX Root Mode. */
5300 VMXDisable();
5301
5302 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5303
5304 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5305 CPUMSetHyperEIP(pVCpu, enmOp);
5306 for (int i = (int)cParams - 1; i >= 0; i--)
5307 CPUMPushHyper(pVCpu, paParam[i]);
5308
5309 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5310
5311 /* Call the switcher. */
5312 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5313 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5314
5315 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5316 /* Make sure the VMX instructions don't cause #UD faults. */
5317 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5318
5319 /* Re-enter VMX Root Mode */
5320 int rc2 = VMXEnable(HCPhysCpuPage);
5321 if (RT_FAILURE(rc2))
5322 {
5323 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5324 ASMSetFlags(fOldEFlags);
5325 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5326 return rc2;
5327 }
5328
5329 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5330 AssertRC(rc2);
5331 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5332 Assert(!(ASMGetFlags() & X86_EFL_IF));
5333 ASMSetFlags(fOldEFlags);
5334 return rc;
5335}
5336
5337
5338/**
5339 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5340 * supporting 64-bit guests.
5341 *
5342 * @returns VBox status code.
5343 * @param fResume Whether to VMLAUNCH or VMRESUME.
5344 * @param pCtx Pointer to the guest-CPU context.
5345 * @param pCache Pointer to the VMCS cache.
5346 * @param pVM The cross context VM structure.
5347 * @param pVCpu The cross context virtual CPU structure.
5348 */
5349DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5350{
5351 NOREF(fResume);
5352
5353 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5354 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5355
5356#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5357 pCache->uPos = 1;
5358 pCache->interPD = PGMGetInterPaeCR3(pVM);
5359 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5360#endif
5361
5362#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5363 pCache->TestIn.HCPhysCpuPage = 0;
5364 pCache->TestIn.HCPhysVmcs = 0;
5365 pCache->TestIn.pCache = 0;
5366 pCache->TestOut.HCPhysVmcs = 0;
5367 pCache->TestOut.pCache = 0;
5368 pCache->TestOut.pCtx = 0;
5369 pCache->TestOut.eflags = 0;
5370#else
5371 NOREF(pCache);
5372#endif
5373
5374 uint32_t aParam[10];
5375 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5376 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5377 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5378 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5379 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5380 aParam[5] = 0;
5381 aParam[6] = VM_RC_ADDR(pVM, pVM);
5382 aParam[7] = 0;
5383 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5384 aParam[9] = 0;
5385
5386#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5387 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5388 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5389#endif
5390 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5391
5392#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5393 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5394 Assert(pCtx->dr[4] == 10);
5395 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5396#endif
5397
5398#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5399 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5400 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5401 pVCpu->hm.s.vmx.HCPhysVmcs));
5402 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5403 pCache->TestOut.HCPhysVmcs));
5404 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5405 pCache->TestOut.pCache));
5406 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5407 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5408 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5409 pCache->TestOut.pCtx));
5410 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5411#endif
5412 return rc;
5413}
5414
5415
5416/**
5417 * Initialize the VMCS-Read cache.
5418 *
5419 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5420 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5421 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5422 * (those that have a 32-bit FULL & HIGH part).
5423 *
5424 * @returns VBox status code.
5425 * @param pVM The cross context VM structure.
5426 * @param pVCpu The cross context virtual CPU structure.
5427 */
5428static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5429{
5430#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5431{ \
5432 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5433 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5434 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5435 ++cReadFields; \
5436}
5437
5438 AssertPtr(pVM);
5439 AssertPtr(pVCpu);
5440 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5441 uint32_t cReadFields = 0;
5442
5443 /*
5444 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5445 * and serve to indicate exceptions to the rules.
5446 */
5447
5448 /* Guest-natural selector base fields. */
5449#if 0
5450 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5453#endif
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5466#if 0
5467 /* Unused natural width guest-state fields. */
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5470#endif
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5472 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5473
5474 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5475#if 0
5476 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5478 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5479 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5480 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5481 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5482 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5485#endif
5486
5487 /* Natural width guest-state fields. */
5488 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5489#if 0
5490 /* Currently unused field. */
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5492#endif
5493
5494 if (pVM->hm.s.fNestedPaging)
5495 {
5496 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5497 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5498 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5499 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5500 }
5501 else
5502 {
5503 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5504 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5505 }
5506
5507#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5508 return VINF_SUCCESS;
5509}
5510
5511
5512/**
5513 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5514 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5515 * darwin, running 64-bit guests).
5516 *
5517 * @returns VBox status code.
5518 * @param pVCpu The cross context virtual CPU structure.
5519 * @param idxField The VMCS field encoding.
5520 * @param u64Val 16, 32 or 64-bit value.
5521 */
5522VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5523{
5524 int rc;
5525 switch (idxField)
5526 {
5527 /*
5528 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5529 */
5530 /* 64-bit Control fields. */
5531 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5532 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5533 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5534 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5535 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5536 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5537 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5538 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5539 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5540 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5541 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5542 case VMX_VMCS64_CTRL_EPTP_FULL:
5543 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5544 /* 64-bit Guest-state fields. */
5545 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5546 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5547 case VMX_VMCS64_GUEST_PAT_FULL:
5548 case VMX_VMCS64_GUEST_EFER_FULL:
5549 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5550 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5551 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5552 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5553 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5554 /* 64-bit Host-state fields. */
5555 case VMX_VMCS64_HOST_PAT_FULL:
5556 case VMX_VMCS64_HOST_EFER_FULL:
5557 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5558 {
5559 rc = VMXWriteVmcs32(idxField, u64Val);
5560 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5561 break;
5562 }
5563
5564 /*
5565 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5566 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5567 */
5568 /* Natural-width Guest-state fields. */
5569 case VMX_VMCS_GUEST_CR3:
5570 case VMX_VMCS_GUEST_ES_BASE:
5571 case VMX_VMCS_GUEST_CS_BASE:
5572 case VMX_VMCS_GUEST_SS_BASE:
5573 case VMX_VMCS_GUEST_DS_BASE:
5574 case VMX_VMCS_GUEST_FS_BASE:
5575 case VMX_VMCS_GUEST_GS_BASE:
5576 case VMX_VMCS_GUEST_LDTR_BASE:
5577 case VMX_VMCS_GUEST_TR_BASE:
5578 case VMX_VMCS_GUEST_GDTR_BASE:
5579 case VMX_VMCS_GUEST_IDTR_BASE:
5580 case VMX_VMCS_GUEST_RSP:
5581 case VMX_VMCS_GUEST_RIP:
5582 case VMX_VMCS_GUEST_SYSENTER_ESP:
5583 case VMX_VMCS_GUEST_SYSENTER_EIP:
5584 {
5585 if (!(u64Val >> 32))
5586 {
5587 /* If this field is 64-bit, VT-x will zero out the top bits. */
5588 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5589 }
5590 else
5591 {
5592 /* Assert that only the 32->64 switcher case should ever come here. */
5593 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5594 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5595 }
5596 break;
5597 }
5598
5599 default:
5600 {
5601 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5602 rc = VERR_INVALID_PARAMETER;
5603 break;
5604 }
5605 }
5606 AssertRCReturn(rc, rc);
5607 return rc;
5608}
5609
5610
5611/**
5612 * Queue up a VMWRITE by using the VMCS write cache.
5613 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5614 *
5615 * @param pVCpu The cross context virtual CPU structure.
5616 * @param idxField The VMCS field encoding.
5617 * @param u64Val 16, 32 or 64-bit value.
5618 */
5619VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5620{
5621 AssertPtr(pVCpu);
5622 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5623
5624 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5625 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5626
5627 /* Make sure there are no duplicates. */
5628 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5629 {
5630 if (pCache->Write.aField[i] == idxField)
5631 {
5632 pCache->Write.aFieldVal[i] = u64Val;
5633 return VINF_SUCCESS;
5634 }
5635 }
5636
5637 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5638 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5639 pCache->Write.cValidEntries++;
5640 return VINF_SUCCESS;
5641}
5642#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5643
5644
5645/**
5646 * Sets up the usage of TSC-offsetting and updates the VMCS.
5647 *
5648 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5649 * VMX preemption timer.
5650 *
5651 * @returns VBox status code.
5652 * @param pVM The cross context VM structure.
5653 * @param pVCpu The cross context virtual CPU structure.
5654 *
5655 * @remarks No-long-jump zone!!!
5656 */
5657static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5658{
5659 int rc;
5660 bool fOffsettedTsc;
5661 bool fParavirtTsc;
5662 if (pVM->hm.s.vmx.fUsePreemptTimer)
5663 {
5664 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5665 &fOffsettedTsc, &fParavirtTsc);
5666
5667 /* Make sure the returned values have sane upper and lower boundaries. */
5668 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5669 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5670 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5671 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5672
5673 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5674 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5675 }
5676 else
5677 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5678
5679 /** @todo later optimize this to be done elsewhere and not before every
5680 * VM-entry. */
5681 if (fParavirtTsc)
5682 {
5683 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5684 information before every VM-entry, hence disable it for performance sake. */
5685#if 0
5686 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5687 AssertRC(rc);
5688#endif
5689 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5690 }
5691
5692 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5693 {
5694 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5695 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5696
5697 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5698 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5699 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5700 }
5701 else
5702 {
5703 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5704 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5705 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5706 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5707 }
5708}
5709
5710
5711/**
5712 * Determines if an exception is a contributory exception.
5713 *
5714 * Contributory exceptions are ones which can cause double-faults unless the
5715 * original exception was a benign exception. Page-fault is intentionally not
5716 * included here as it's a conditional contributory exception.
5717 *
5718 * @returns true if the exception is contributory, false otherwise.
5719 * @param uVector The exception vector.
5720 */
5721DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5722{
5723 switch (uVector)
5724 {
5725 case X86_XCPT_GP:
5726 case X86_XCPT_SS:
5727 case X86_XCPT_NP:
5728 case X86_XCPT_TS:
5729 case X86_XCPT_DE:
5730 return true;
5731 default:
5732 break;
5733 }
5734 return false;
5735}
5736
5737
5738/**
5739 * Sets an event as a pending event to be injected into the guest.
5740 *
5741 * @param pVCpu The cross context virtual CPU structure.
5742 * @param u32IntInfo The VM-entry interruption-information field.
5743 * @param cbInstr The VM-entry instruction length in bytes (for software
5744 * interrupts, exceptions and privileged software
5745 * exceptions).
5746 * @param u32ErrCode The VM-entry exception error code.
5747 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5748 * page-fault.
5749 *
5750 * @remarks Statistics counter assumes this is a guest event being injected or
5751 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5752 * always incremented.
5753 */
5754DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5755 RTGCUINTPTR GCPtrFaultAddress)
5756{
5757 Assert(!pVCpu->hm.s.Event.fPending);
5758 pVCpu->hm.s.Event.fPending = true;
5759 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5760 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5761 pVCpu->hm.s.Event.cbInstr = cbInstr;
5762 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5763}
5764
5765
5766/**
5767 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5768 *
5769 * @param pVCpu The cross context virtual CPU structure.
5770 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5771 * out-of-sync. Make sure to update the required fields
5772 * before using them.
5773 */
5774DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5775{
5776 NOREF(pMixedCtx);
5777 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5778 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5779 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5780 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5781}
5782
5783
5784/**
5785 * Handle a condition that occurred while delivering an event through the guest
5786 * IDT.
5787 *
5788 * @returns Strict VBox status code (i.e. informational status codes too).
5789 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5790 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5791 * to continue execution of the guest which will delivery the \#DF.
5792 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5793 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5794 *
5795 * @param pVCpu The cross context virtual CPU structure.
5796 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5797 * out-of-sync. Make sure to update the required fields
5798 * before using them.
5799 * @param pVmxTransient Pointer to the VMX transient structure.
5800 *
5801 * @remarks No-long-jump zone!!!
5802 */
5803static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5804{
5805 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5806
5807 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5808 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5809
5810 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5811 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5812 {
5813 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5814 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5815
5816 typedef enum
5817 {
5818 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5819 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5820 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5821 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5822 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5823 } VMXREFLECTXCPT;
5824
5825 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5826 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5827 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5828 {
5829 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5830 {
5831 enmReflect = VMXREFLECTXCPT_XCPT;
5832#ifdef VBOX_STRICT
5833 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5834 && uExitVector == X86_XCPT_PF)
5835 {
5836 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5837 }
5838#endif
5839 if ( uExitVector == X86_XCPT_PF
5840 && uIdtVector == X86_XCPT_PF)
5841 {
5842 pVmxTransient->fVectoringDoublePF = true;
5843 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5844 }
5845 else if ( uExitVector == X86_XCPT_AC
5846 && uIdtVector == X86_XCPT_AC)
5847 {
5848 enmReflect = VMXREFLECTXCPT_HANG;
5849 Log4(("IDT: Nested #AC - Bad guest\n"));
5850 }
5851 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5852 && hmR0VmxIsContributoryXcpt(uExitVector)
5853 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5854 || uIdtVector == X86_XCPT_PF))
5855 {
5856 enmReflect = VMXREFLECTXCPT_DF;
5857 }
5858 else if (uIdtVector == X86_XCPT_DF)
5859 enmReflect = VMXREFLECTXCPT_TF;
5860 }
5861 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5862 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5863 {
5864 /*
5865 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5866 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5867 */
5868 enmReflect = VMXREFLECTXCPT_XCPT;
5869
5870 if (uExitVector == X86_XCPT_PF)
5871 {
5872 pVmxTransient->fVectoringPF = true;
5873 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5874 }
5875 }
5876 }
5877 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5878 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5879 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5880 {
5881 /*
5882 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5883 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5884 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5885 */
5886 enmReflect = VMXREFLECTXCPT_XCPT;
5887 }
5888
5889 /*
5890 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5891 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5892 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5893 *
5894 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5895 */
5896 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5897 && enmReflect == VMXREFLECTXCPT_XCPT
5898 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5899 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5900 {
5901 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5902 }
5903
5904 switch (enmReflect)
5905 {
5906 case VMXREFLECTXCPT_XCPT:
5907 {
5908 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5909 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5910 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5911
5912 uint32_t u32ErrCode = 0;
5913 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5914 {
5915 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5916 AssertRCReturn(rc2, rc2);
5917 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5918 }
5919
5920 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5921 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5922 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5923 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5924 rcStrict = VINF_SUCCESS;
5925 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5926 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5927
5928 break;
5929 }
5930
5931 case VMXREFLECTXCPT_DF:
5932 {
5933 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5934 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5935 rcStrict = VINF_HM_DOUBLE_FAULT;
5936 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5937 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5938
5939 break;
5940 }
5941
5942 case VMXREFLECTXCPT_TF:
5943 {
5944 rcStrict = VINF_EM_RESET;
5945 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5946 uExitVector));
5947 break;
5948 }
5949
5950 case VMXREFLECTXCPT_HANG:
5951 {
5952 rcStrict = VERR_EM_GUEST_CPU_HANG;
5953 break;
5954 }
5955
5956 default:
5957 Assert(rcStrict == VINF_SUCCESS);
5958 break;
5959 }
5960 }
5961 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5962 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5963 && uExitVector != X86_XCPT_DF
5964 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5965 {
5966 /*
5967 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5968 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5969 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5970 */
5971 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5972 {
5973 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5974 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5975 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5976 }
5977 }
5978
5979 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5980 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5981 return rcStrict;
5982}
5983
5984
5985/**
5986 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5987 *
5988 * @returns VBox status code.
5989 * @param pVCpu The cross context virtual CPU structure.
5990 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5991 * out-of-sync. Make sure to update the required fields
5992 * before using them.
5993 *
5994 * @remarks No-long-jump zone!!!
5995 */
5996static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5997{
5998 NOREF(pMixedCtx);
5999
6000 /*
6001 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6002 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6003 */
6004 VMMRZCallRing3Disable(pVCpu);
6005 HM_DISABLE_PREEMPT();
6006
6007 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6008 {
6009 uint32_t uVal = 0;
6010 uint32_t uShadow = 0;
6011 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6012 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6013 AssertRCReturn(rc, rc);
6014
6015 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6016 CPUMSetGuestCR0(pVCpu, uVal);
6017 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6018 }
6019
6020 HM_RESTORE_PREEMPT();
6021 VMMRZCallRing3Enable(pVCpu);
6022 return VINF_SUCCESS;
6023}
6024
6025
6026/**
6027 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6028 *
6029 * @returns VBox status code.
6030 * @param pVCpu The cross context virtual CPU structure.
6031 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6032 * out-of-sync. Make sure to update the required fields
6033 * before using them.
6034 *
6035 * @remarks No-long-jump zone!!!
6036 */
6037static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6038{
6039 NOREF(pMixedCtx);
6040
6041 int rc = VINF_SUCCESS;
6042 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6043 {
6044 uint32_t uVal = 0;
6045 uint32_t uShadow = 0;
6046 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6047 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6048 AssertRCReturn(rc, rc);
6049
6050 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6051 CPUMSetGuestCR4(pVCpu, uVal);
6052 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6053 }
6054 return rc;
6055}
6056
6057
6058/**
6059 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6060 *
6061 * @returns VBox status code.
6062 * @param pVCpu The cross context virtual CPU structure.
6063 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6064 * out-of-sync. Make sure to update the required fields
6065 * before using them.
6066 *
6067 * @remarks No-long-jump zone!!!
6068 */
6069static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6070{
6071 int rc = VINF_SUCCESS;
6072 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6073 {
6074 uint64_t u64Val = 0;
6075 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6076 AssertRCReturn(rc, rc);
6077
6078 pMixedCtx->rip = u64Val;
6079 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6080 }
6081 return rc;
6082}
6083
6084
6085/**
6086 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6087 *
6088 * @returns VBox status code.
6089 * @param pVCpu The cross context virtual CPU structure.
6090 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6091 * out-of-sync. Make sure to update the required fields
6092 * before using them.
6093 *
6094 * @remarks No-long-jump zone!!!
6095 */
6096static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6097{
6098 int rc = VINF_SUCCESS;
6099 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6100 {
6101 uint64_t u64Val = 0;
6102 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6103 AssertRCReturn(rc, rc);
6104
6105 pMixedCtx->rsp = u64Val;
6106 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6107 }
6108 return rc;
6109}
6110
6111
6112/**
6113 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6114 *
6115 * @returns VBox status code.
6116 * @param pVCpu The cross context virtual CPU structure.
6117 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6118 * out-of-sync. Make sure to update the required fields
6119 * before using them.
6120 *
6121 * @remarks No-long-jump zone!!!
6122 */
6123static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6124{
6125 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6126 {
6127 uint32_t uVal = 0;
6128 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6129 AssertRCReturn(rc, rc);
6130
6131 pMixedCtx->eflags.u32 = uVal;
6132 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6133 {
6134 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6135 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6136
6137 pMixedCtx->eflags.Bits.u1VM = 0;
6138 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6139 }
6140
6141 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6142 }
6143 return VINF_SUCCESS;
6144}
6145
6146
6147/**
6148 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6149 * guest-CPU context.
6150 */
6151DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6152{
6153 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6154 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6155 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6156 return rc;
6157}
6158
6159
6160/**
6161 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6162 * from the guest-state area in the VMCS.
6163 *
6164 * @param pVCpu The cross context virtual CPU structure.
6165 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6166 * out-of-sync. Make sure to update the required fields
6167 * before using them.
6168 *
6169 * @remarks No-long-jump zone!!!
6170 */
6171static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6172{
6173 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6174 {
6175 uint32_t uIntrState = 0;
6176 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6177 AssertRC(rc);
6178
6179 if (!uIntrState)
6180 {
6181 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6182 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6183
6184 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6185 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6186 }
6187 else
6188 {
6189 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6190 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6191 {
6192 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6193 AssertRC(rc);
6194 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6195 AssertRC(rc);
6196
6197 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6198 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6199 }
6200 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6201 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6202
6203 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6204 {
6205 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6206 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6207 }
6208 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6209 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6210 }
6211
6212 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6213 }
6214}
6215
6216
6217/**
6218 * Saves the guest's activity state.
6219 *
6220 * @returns VBox status code.
6221 * @param pVCpu The cross context virtual CPU structure.
6222 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6223 * out-of-sync. Make sure to update the required fields
6224 * before using them.
6225 *
6226 * @remarks No-long-jump zone!!!
6227 */
6228static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6229{
6230 NOREF(pMixedCtx);
6231 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6232 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6233 return VINF_SUCCESS;
6234}
6235
6236
6237/**
6238 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6239 * the current VMCS into the guest-CPU context.
6240 *
6241 * @returns VBox status code.
6242 * @param pVCpu The cross context virtual CPU structure.
6243 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6244 * out-of-sync. Make sure to update the required fields
6245 * before using them.
6246 *
6247 * @remarks No-long-jump zone!!!
6248 */
6249static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6250{
6251 int rc = VINF_SUCCESS;
6252 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6253 {
6254 uint32_t u32Val = 0;
6255 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6256 pMixedCtx->SysEnter.cs = u32Val;
6257 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6258 }
6259
6260 uint64_t u64Val = 0;
6261 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6262 {
6263 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6264 pMixedCtx->SysEnter.eip = u64Val;
6265 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6266 }
6267 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6268 {
6269 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6270 pMixedCtx->SysEnter.esp = u64Val;
6271 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6272 }
6273 return rc;
6274}
6275
6276
6277/**
6278 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6279 * the CPU back into the guest-CPU context.
6280 *
6281 * @returns VBox status code.
6282 * @param pVCpu The cross context virtual CPU structure.
6283 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6284 * out-of-sync. Make sure to update the required fields
6285 * before using them.
6286 *
6287 * @remarks No-long-jump zone!!!
6288 */
6289static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6290{
6291 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6292 VMMRZCallRing3Disable(pVCpu);
6293 HM_DISABLE_PREEMPT();
6294
6295 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6296 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6297 {
6298 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6299 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6300 }
6301
6302 HM_RESTORE_PREEMPT();
6303 VMMRZCallRing3Enable(pVCpu);
6304
6305 return VINF_SUCCESS;
6306}
6307
6308
6309/**
6310 * Saves the auto load/store'd guest MSRs from the current VMCS into
6311 * the guest-CPU context.
6312 *
6313 * @returns VBox status code.
6314 * @param pVCpu The cross context virtual CPU structure.
6315 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6316 * out-of-sync. Make sure to update the required fields
6317 * before using them.
6318 *
6319 * @remarks No-long-jump zone!!!
6320 */
6321static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6322{
6323 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6324 return VINF_SUCCESS;
6325
6326 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6327 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6328 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6329 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6330 {
6331 switch (pMsr->u32Msr)
6332 {
6333 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6334 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6335 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6336 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6337 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6338 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6339 break;
6340
6341 default:
6342 {
6343 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6344 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6345 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6346 }
6347 }
6348 }
6349
6350 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6351 return VINF_SUCCESS;
6352}
6353
6354
6355/**
6356 * Saves the guest control registers from the current VMCS into the guest-CPU
6357 * context.
6358 *
6359 * @returns VBox status code.
6360 * @param pVCpu The cross context virtual CPU structure.
6361 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6362 * out-of-sync. Make sure to update the required fields
6363 * before using them.
6364 *
6365 * @remarks No-long-jump zone!!!
6366 */
6367static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6368{
6369 /* Guest CR0. Guest FPU. */
6370 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6371 AssertRCReturn(rc, rc);
6372
6373 /* Guest CR4. */
6374 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6375 AssertRCReturn(rc, rc);
6376
6377 /* Guest CR2 - updated always during the world-switch or in #PF. */
6378 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6379 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6380 {
6381 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6382 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6383
6384 PVM pVM = pVCpu->CTX_SUFF(pVM);
6385 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6386 || ( pVM->hm.s.fNestedPaging
6387 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6388 {
6389 uint64_t u64Val = 0;
6390 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6391 if (pMixedCtx->cr3 != u64Val)
6392 {
6393 CPUMSetGuestCR3(pVCpu, u64Val);
6394 if (VMMRZCallRing3IsEnabled(pVCpu))
6395 {
6396 PGMUpdateCR3(pVCpu, u64Val);
6397 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6398 }
6399 else
6400 {
6401 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6402 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6403 }
6404 }
6405
6406 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6407 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6408 {
6409 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6410 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6411 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6412 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6413 AssertRCReturn(rc, rc);
6414
6415 if (VMMRZCallRing3IsEnabled(pVCpu))
6416 {
6417 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6418 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6419 }
6420 else
6421 {
6422 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6423 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6424 }
6425 }
6426 }
6427
6428 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6429 }
6430
6431 /*
6432 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6433 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6434 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6435 *
6436 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6437 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6438 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6439 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6440 *
6441 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6442 */
6443 if (VMMRZCallRing3IsEnabled(pVCpu))
6444 {
6445 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6446 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6447
6448 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6449 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6450
6451 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6452 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6453 }
6454
6455 return rc;
6456}
6457
6458
6459/**
6460 * Reads a guest segment register from the current VMCS into the guest-CPU
6461 * context.
6462 *
6463 * @returns VBox status code.
6464 * @param pVCpu The cross context virtual CPU structure.
6465 * @param idxSel Index of the selector in the VMCS.
6466 * @param idxLimit Index of the segment limit in the VMCS.
6467 * @param idxBase Index of the segment base in the VMCS.
6468 * @param idxAccess Index of the access rights of the segment in the VMCS.
6469 * @param pSelReg Pointer to the segment selector.
6470 *
6471 * @remarks No-long-jump zone!!!
6472 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6473 * macro as that takes care of whether to read from the VMCS cache or
6474 * not.
6475 */
6476DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6477 PCPUMSELREG pSelReg)
6478{
6479 NOREF(pVCpu);
6480
6481 uint32_t u32Val = 0;
6482 int rc = VMXReadVmcs32(idxSel, &u32Val);
6483 AssertRCReturn(rc, rc);
6484 pSelReg->Sel = (uint16_t)u32Val;
6485 pSelReg->ValidSel = (uint16_t)u32Val;
6486 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6487
6488 rc = VMXReadVmcs32(idxLimit, &u32Val);
6489 AssertRCReturn(rc, rc);
6490 pSelReg->u32Limit = u32Val;
6491
6492 uint64_t u64Val = 0;
6493 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6494 AssertRCReturn(rc, rc);
6495 pSelReg->u64Base = u64Val;
6496
6497 rc = VMXReadVmcs32(idxAccess, &u32Val);
6498 AssertRCReturn(rc, rc);
6499 pSelReg->Attr.u = u32Val;
6500
6501 /*
6502 * If VT-x marks the segment as unusable, most other bits remain undefined:
6503 * - For CS the L, D and G bits have meaning.
6504 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6505 * - For the remaining data segments no bits are defined.
6506 *
6507 * The present bit and the unusable bit has been observed to be set at the
6508 * same time (the selector was supposed to be invalid as we started executing
6509 * a V8086 interrupt in ring-0).
6510 *
6511 * What should be important for the rest of the VBox code, is that the P bit is
6512 * cleared. Some of the other VBox code recognizes the unusable bit, but
6513 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6514 * safe side here, we'll strip off P and other bits we don't care about. If
6515 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6516 *
6517 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6518 */
6519 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6520 {
6521 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6522
6523 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6524 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6525 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6526
6527 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6528#ifdef DEBUG_bird
6529 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6530 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6531 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6532#endif
6533 }
6534 return VINF_SUCCESS;
6535}
6536
6537
6538#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6539# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6540 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6541 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6542#else
6543# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6544 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6545 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6546#endif
6547
6548
6549/**
6550 * Saves the guest segment registers from the current VMCS into the guest-CPU
6551 * context.
6552 *
6553 * @returns VBox status code.
6554 * @param pVCpu The cross context virtual CPU structure.
6555 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6556 * out-of-sync. Make sure to update the required fields
6557 * before using them.
6558 *
6559 * @remarks No-long-jump zone!!!
6560 */
6561static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6562{
6563 /* Guest segment registers. */
6564 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6565 {
6566 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6567 AssertRCReturn(rc, rc);
6568
6569 rc = VMXLOCAL_READ_SEG(CS, cs);
6570 rc |= VMXLOCAL_READ_SEG(SS, ss);
6571 rc |= VMXLOCAL_READ_SEG(DS, ds);
6572 rc |= VMXLOCAL_READ_SEG(ES, es);
6573 rc |= VMXLOCAL_READ_SEG(FS, fs);
6574 rc |= VMXLOCAL_READ_SEG(GS, gs);
6575 AssertRCReturn(rc, rc);
6576
6577 /* Restore segment attributes for real-on-v86 mode hack. */
6578 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6579 {
6580 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6581 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6582 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6583 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6584 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6585 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6586 }
6587 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6588 }
6589
6590 return VINF_SUCCESS;
6591}
6592
6593
6594/**
6595 * Saves the guest descriptor table registers and task register from the current
6596 * VMCS into the guest-CPU context.
6597 *
6598 * @returns VBox status code.
6599 * @param pVCpu The cross context virtual CPU structure.
6600 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6601 * out-of-sync. Make sure to update the required fields
6602 * before using them.
6603 *
6604 * @remarks No-long-jump zone!!!
6605 */
6606static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6607{
6608 int rc = VINF_SUCCESS;
6609
6610 /* Guest LDTR. */
6611 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6612 {
6613 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6614 AssertRCReturn(rc, rc);
6615 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6616 }
6617
6618 /* Guest GDTR. */
6619 uint64_t u64Val = 0;
6620 uint32_t u32Val = 0;
6621 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6622 {
6623 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6624 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6625 pMixedCtx->gdtr.pGdt = u64Val;
6626 pMixedCtx->gdtr.cbGdt = u32Val;
6627 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6628 }
6629
6630 /* Guest IDTR. */
6631 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6632 {
6633 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6634 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6635 pMixedCtx->idtr.pIdt = u64Val;
6636 pMixedCtx->idtr.cbIdt = u32Val;
6637 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6638 }
6639
6640 /* Guest TR. */
6641 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6642 {
6643 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6644 AssertRCReturn(rc, rc);
6645
6646 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6647 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6648 {
6649 rc = VMXLOCAL_READ_SEG(TR, tr);
6650 AssertRCReturn(rc, rc);
6651 }
6652 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6653 }
6654 return rc;
6655}
6656
6657#undef VMXLOCAL_READ_SEG
6658
6659
6660/**
6661 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6662 * context.
6663 *
6664 * @returns VBox status code.
6665 * @param pVCpu The cross context virtual CPU structure.
6666 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6667 * out-of-sync. Make sure to update the required fields
6668 * before using them.
6669 *
6670 * @remarks No-long-jump zone!!!
6671 */
6672static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6673{
6674 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6675 {
6676 if (!pVCpu->hm.s.fUsingHyperDR7)
6677 {
6678 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6679 uint32_t u32Val;
6680 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6681 pMixedCtx->dr[7] = u32Val;
6682 }
6683
6684 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6685 }
6686 return VINF_SUCCESS;
6687}
6688
6689
6690/**
6691 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6692 *
6693 * @returns VBox status code.
6694 * @param pVCpu The cross context virtual CPU structure.
6695 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6696 * out-of-sync. Make sure to update the required fields
6697 * before using them.
6698 *
6699 * @remarks No-long-jump zone!!!
6700 */
6701static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6702{
6703 NOREF(pMixedCtx);
6704
6705 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6706 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6707 return VINF_SUCCESS;
6708}
6709
6710
6711/**
6712 * Saves the entire guest state from the currently active VMCS into the
6713 * guest-CPU context.
6714 *
6715 * This essentially VMREADs all guest-data.
6716 *
6717 * @returns VBox status code.
6718 * @param pVCpu The cross context virtual CPU structure.
6719 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6720 * out-of-sync. Make sure to update the required fields
6721 * before using them.
6722 */
6723static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6724{
6725 Assert(pVCpu);
6726 Assert(pMixedCtx);
6727
6728 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6729 return VINF_SUCCESS;
6730
6731 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6732 again on the ring-3 callback path, there is no real need to. */
6733 if (VMMRZCallRing3IsEnabled(pVCpu))
6734 VMMR0LogFlushDisable(pVCpu);
6735 else
6736 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6737 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6738
6739 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6740 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6741
6742 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6743 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6744
6745 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6746 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6747
6748 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6749 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6750
6751 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6752 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6753
6754 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6755 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6756
6757 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6758 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6759
6760 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6761 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6762
6763 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6764 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6765
6766 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6767 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6768
6769 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6770 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6771 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6772
6773 if (VMMRZCallRing3IsEnabled(pVCpu))
6774 VMMR0LogFlushEnable(pVCpu);
6775
6776 return VINF_SUCCESS;
6777}
6778
6779
6780/**
6781 * Saves basic guest registers needed for IEM instruction execution.
6782 *
6783 * @returns VBox status code (OR-able).
6784 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6785 * @param pMixedCtx Pointer to the CPU context of the guest.
6786 * @param fMemory Whether the instruction being executed operates on
6787 * memory or not. Only CR0 is synced up if clear.
6788 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6789 */
6790static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6791{
6792 /*
6793 * We assume all general purpose registers other than RSP are available.
6794 *
6795 * RIP is a must, as it will be incremented or otherwise changed.
6796 *
6797 * RFLAGS are always required to figure the CPL.
6798 *
6799 * RSP isn't always required, however it's a GPR, so frequently required.
6800 *
6801 * SS and CS are the only segment register needed if IEM doesn't do memory
6802 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6803 *
6804 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6805 * be required for memory accesses.
6806 *
6807 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6808 */
6809 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6810 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6811 if (fNeedRsp)
6812 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6813 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6814 if (!fMemory)
6815 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6816 else
6817 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6818 AssertRCReturn(rc, rc);
6819 return rc;
6820}
6821
6822
6823/**
6824 * Ensures that we've got a complete basic guest-context.
6825 *
6826 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6827 * is for the interpreter.
6828 *
6829 * @returns VBox status code.
6830 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6831 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6832 * needing to be synced in.
6833 * @thread EMT(pVCpu)
6834 */
6835VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6836{
6837 /* Note! Since this is only applicable to VT-x, the implementation is placed
6838 in the VT-x part of the sources instead of the generic stuff. */
6839 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6840 {
6841 /* For now, imply that the caller might change everything too. */
6842 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6843 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6844 }
6845 return VINF_SUCCESS;
6846}
6847
6848
6849/**
6850 * Check per-VM and per-VCPU force flag actions that require us to go back to
6851 * ring-3 for one reason or another.
6852 *
6853 * @returns Strict VBox status code (i.e. informational status codes too)
6854 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6855 * ring-3.
6856 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6857 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6858 * interrupts)
6859 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6860 * all EMTs to be in ring-3.
6861 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6862 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6863 * to the EM loop.
6864 *
6865 * @param pVM The cross context VM structure.
6866 * @param pVCpu The cross context virtual CPU structure.
6867 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6868 * out-of-sync. Make sure to update the required fields
6869 * before using them.
6870 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6871 */
6872static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6873{
6874 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6875
6876 /*
6877 * Anything pending? Should be more likely than not if we're doing a good job.
6878 */
6879 if ( !fStepping
6880 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6881 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6882 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6883 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6884 return VINF_SUCCESS;
6885
6886 /* We need the control registers now, make sure the guest-CPU context is updated. */
6887 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6888 AssertRCReturn(rc3, rc3);
6889
6890 /* Pending HM CR3 sync. */
6891 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6892 {
6893 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6894 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6895 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6896 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6897 }
6898
6899 /* Pending HM PAE PDPEs. */
6900 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6901 {
6902 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6903 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6904 }
6905
6906 /* Pending PGM C3 sync. */
6907 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6908 {
6909 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6910 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6911 if (rcStrict2 != VINF_SUCCESS)
6912 {
6913 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6914 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6915 return rcStrict2;
6916 }
6917 }
6918
6919 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6920 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6921 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6922 {
6923 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6924 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6925 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6926 return rc2;
6927 }
6928
6929 /* Pending VM request packets, such as hardware interrupts. */
6930 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6931 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6932 {
6933 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6934 return VINF_EM_PENDING_REQUEST;
6935 }
6936
6937 /* Pending PGM pool flushes. */
6938 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6939 {
6940 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6941 return VINF_PGM_POOL_FLUSH_PENDING;
6942 }
6943
6944 /* Pending DMA requests. */
6945 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6946 {
6947 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6948 return VINF_EM_RAW_TO_R3;
6949 }
6950
6951 return VINF_SUCCESS;
6952}
6953
6954
6955/**
6956 * Converts any TRPM trap into a pending HM event. This is typically used when
6957 * entering from ring-3 (not longjmp returns).
6958 *
6959 * @param pVCpu The cross context virtual CPU structure.
6960 */
6961static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6962{
6963 Assert(TRPMHasTrap(pVCpu));
6964 Assert(!pVCpu->hm.s.Event.fPending);
6965
6966 uint8_t uVector;
6967 TRPMEVENT enmTrpmEvent;
6968 RTGCUINT uErrCode;
6969 RTGCUINTPTR GCPtrFaultAddress;
6970 uint8_t cbInstr;
6971
6972 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6973 AssertRC(rc);
6974
6975 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6976 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6977 if (enmTrpmEvent == TRPM_TRAP)
6978 {
6979 switch (uVector)
6980 {
6981 case X86_XCPT_NMI:
6982 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6983 break;
6984
6985 case X86_XCPT_BP:
6986 case X86_XCPT_OF:
6987 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6988 break;
6989
6990 case X86_XCPT_PF:
6991 case X86_XCPT_DF:
6992 case X86_XCPT_TS:
6993 case X86_XCPT_NP:
6994 case X86_XCPT_SS:
6995 case X86_XCPT_GP:
6996 case X86_XCPT_AC:
6997 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6998 /* no break! */
6999 default:
7000 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7001 break;
7002 }
7003 }
7004 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7005 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7006 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7007 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7008 else
7009 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7010
7011 rc = TRPMResetTrap(pVCpu);
7012 AssertRC(rc);
7013 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7014 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7015
7016 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7017}
7018
7019
7020/**
7021 * Converts the pending HM event into a TRPM trap.
7022 *
7023 * @param pVCpu The cross context virtual CPU structure.
7024 */
7025static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7026{
7027 Assert(pVCpu->hm.s.Event.fPending);
7028
7029 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7030 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7031 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7032 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7033
7034 /* If a trap was already pending, we did something wrong! */
7035 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7036
7037 TRPMEVENT enmTrapType;
7038 switch (uVectorType)
7039 {
7040 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7041 enmTrapType = TRPM_HARDWARE_INT;
7042 break;
7043
7044 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7045 enmTrapType = TRPM_SOFTWARE_INT;
7046 break;
7047
7048 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7049 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7050 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7051 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7052 enmTrapType = TRPM_TRAP;
7053 break;
7054
7055 default:
7056 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7057 enmTrapType = TRPM_32BIT_HACK;
7058 break;
7059 }
7060
7061 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7062
7063 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7064 AssertRC(rc);
7065
7066 if (fErrorCodeValid)
7067 TRPMSetErrorCode(pVCpu, uErrorCode);
7068
7069 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7070 && uVector == X86_XCPT_PF)
7071 {
7072 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7073 }
7074 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7075 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7076 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7077 {
7078 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7079 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7080 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7081 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7082 }
7083
7084 /* Clear any pending events from the VMCS. */
7085 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7086 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7087
7088 /* We're now done converting the pending event. */
7089 pVCpu->hm.s.Event.fPending = false;
7090}
7091
7092
7093/**
7094 * Does the necessary state syncing before returning to ring-3 for any reason
7095 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7096 *
7097 * @returns VBox status code.
7098 * @param pVM The cross context VM structure.
7099 * @param pVCpu The cross context virtual CPU structure.
7100 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7101 * be out-of-sync. Make sure to update the required
7102 * fields before using them.
7103 * @param fSaveGuestState Whether to save the guest state or not.
7104 *
7105 * @remarks No-long-jmp zone!!!
7106 */
7107static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7108{
7109 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7110 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7111
7112 RTCPUID idCpu = RTMpCpuId();
7113 Log4Func(("HostCpuId=%u\n", idCpu));
7114
7115 /*
7116 * !!! IMPORTANT !!!
7117 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7118 */
7119
7120 /* Save the guest state if necessary. */
7121 if ( fSaveGuestState
7122 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7123 {
7124 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7125 AssertRCReturn(rc, rc);
7126 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7127 }
7128
7129 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7130 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7131 {
7132 if (fSaveGuestState)
7133 {
7134 /* We shouldn't reload CR0 without saving it first. */
7135 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7136 AssertRCReturn(rc, rc);
7137 }
7138 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7139 }
7140
7141 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7142#ifdef VBOX_STRICT
7143 if (CPUMIsHyperDebugStateActive(pVCpu))
7144 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7145#endif
7146 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7147 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7148 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7149 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7150
7151#if HC_ARCH_BITS == 64
7152 /* Restore host-state bits that VT-x only restores partially. */
7153 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7154 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7155 {
7156 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7157 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7158 }
7159 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7160#endif
7161
7162 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7163 if (pVCpu->hm.s.vmx.fLazyMsrs)
7164 {
7165 /* We shouldn't reload the guest MSRs without saving it first. */
7166 if (!fSaveGuestState)
7167 {
7168 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7169 AssertRCReturn(rc, rc);
7170 }
7171 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7172 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7173 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7174 }
7175
7176 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7177 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7178
7179 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7180 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7181 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7182 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7183 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7184 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7185 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7186 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7187
7188 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7189
7190 /** @todo This partially defeats the purpose of having preemption hooks.
7191 * The problem is, deregistering the hooks should be moved to a place that
7192 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7193 * context.
7194 */
7195 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7196 {
7197 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7198 AssertRCReturn(rc, rc);
7199
7200 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7201 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7202 }
7203 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7204 NOREF(idCpu);
7205
7206 return VINF_SUCCESS;
7207}
7208
7209
7210/**
7211 * Leaves the VT-x session.
7212 *
7213 * @returns VBox status code.
7214 * @param pVM The cross context VM structure.
7215 * @param pVCpu The cross context virtual CPU structure.
7216 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7217 * out-of-sync. Make sure to update the required fields
7218 * before using them.
7219 *
7220 * @remarks No-long-jmp zone!!!
7221 */
7222DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7223{
7224 HM_DISABLE_PREEMPT();
7225 HMVMX_ASSERT_CPU_SAFE();
7226 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7227 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7228
7229 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7230 and done this from the VMXR0ThreadCtxCallback(). */
7231 if (!pVCpu->hm.s.fLeaveDone)
7232 {
7233 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7234 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7235 pVCpu->hm.s.fLeaveDone = true;
7236 }
7237 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7238
7239 /*
7240 * !!! IMPORTANT !!!
7241 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7242 */
7243
7244 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7245 /** @todo Deregistering here means we need to VMCLEAR always
7246 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7247 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7248 VMMR0ThreadCtxHookDisable(pVCpu);
7249
7250 /* Leave HM context. This takes care of local init (term). */
7251 int rc = HMR0LeaveCpu(pVCpu);
7252
7253 HM_RESTORE_PREEMPT();
7254 return rc;
7255}
7256
7257
7258/**
7259 * Does the necessary state syncing before doing a longjmp to ring-3.
7260 *
7261 * @returns VBox status code.
7262 * @param pVM The cross context VM structure.
7263 * @param pVCpu The cross context virtual CPU structure.
7264 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7265 * out-of-sync. Make sure to update the required fields
7266 * before using them.
7267 *
7268 * @remarks No-long-jmp zone!!!
7269 */
7270DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7271{
7272 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7273}
7274
7275
7276/**
7277 * Take necessary actions before going back to ring-3.
7278 *
7279 * An action requires us to go back to ring-3. This function does the necessary
7280 * steps before we can safely return to ring-3. This is not the same as longjmps
7281 * to ring-3, this is voluntary and prepares the guest so it may continue
7282 * executing outside HM (recompiler/IEM).
7283 *
7284 * @returns VBox status code.
7285 * @param pVM The cross context VM structure.
7286 * @param pVCpu The cross context virtual CPU structure.
7287 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7288 * out-of-sync. Make sure to update the required fields
7289 * before using them.
7290 * @param rcExit The reason for exiting to ring-3. Can be
7291 * VINF_VMM_UNKNOWN_RING3_CALL.
7292 */
7293static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7294{
7295 Assert(pVM);
7296 Assert(pVCpu);
7297 Assert(pMixedCtx);
7298 HMVMX_ASSERT_PREEMPT_SAFE();
7299
7300 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7301 {
7302 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7303 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7304 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7305 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7306 }
7307
7308 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7309 VMMRZCallRing3Disable(pVCpu);
7310 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7311
7312 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7313 if (pVCpu->hm.s.Event.fPending)
7314 {
7315 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7316 Assert(!pVCpu->hm.s.Event.fPending);
7317 }
7318
7319 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7320 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7321
7322 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7323 and if we're injecting an event we should have a TRPM trap pending. */
7324 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7325#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7326 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7327#endif
7328
7329 /* Save guest state and restore host state bits. */
7330 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7331 AssertRCReturn(rc, rc);
7332 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7333 /* Thread-context hooks are unregistered at this point!!! */
7334
7335 /* Sync recompiler state. */
7336 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7337 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7338 | CPUM_CHANGED_LDTR
7339 | CPUM_CHANGED_GDTR
7340 | CPUM_CHANGED_IDTR
7341 | CPUM_CHANGED_TR
7342 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7343 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7344 if ( pVM->hm.s.fNestedPaging
7345 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7346 {
7347 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7348 }
7349
7350 Assert(!pVCpu->hm.s.fClearTrapFlag);
7351
7352 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7353 if (rcExit != VINF_EM_RAW_INTERRUPT)
7354 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7355
7356 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7357
7358 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7359 VMMRZCallRing3RemoveNotification(pVCpu);
7360 VMMRZCallRing3Enable(pVCpu);
7361
7362 return rc;
7363}
7364
7365
7366/**
7367 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7368 * longjump to ring-3 and possibly get preempted.
7369 *
7370 * @returns VBox status code.
7371 * @param pVCpu The cross context virtual CPU structure.
7372 * @param enmOperation The operation causing the ring-3 longjump.
7373 * @param pvUser Opaque pointer to the guest-CPU context. The data
7374 * may be out-of-sync. Make sure to update the required
7375 * fields before using them.
7376 */
7377static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7378{
7379 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7380 {
7381 /*
7382 * !!! IMPORTANT !!!
7383 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7384 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7385 */
7386 VMMRZCallRing3RemoveNotification(pVCpu);
7387 VMMRZCallRing3Disable(pVCpu);
7388 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7389 RTThreadPreemptDisable(&PreemptState);
7390
7391 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7392 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7393
7394#if HC_ARCH_BITS == 64
7395 /* Restore host-state bits that VT-x only restores partially. */
7396 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7397 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7398 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7399 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7400#endif
7401 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7402 if (pVCpu->hm.s.vmx.fLazyMsrs)
7403 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7404
7405 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7406 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7407 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7408 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7409 {
7410 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7411 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7412 }
7413
7414 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7415 VMMR0ThreadCtxHookDisable(pVCpu);
7416 HMR0LeaveCpu(pVCpu);
7417 RTThreadPreemptRestore(&PreemptState);
7418 return VINF_SUCCESS;
7419 }
7420
7421 Assert(pVCpu);
7422 Assert(pvUser);
7423 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7424 HMVMX_ASSERT_PREEMPT_SAFE();
7425
7426 VMMRZCallRing3Disable(pVCpu);
7427 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7428
7429 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7430 enmOperation));
7431
7432 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7433 AssertRCReturn(rc, rc);
7434
7435 VMMRZCallRing3Enable(pVCpu);
7436 return VINF_SUCCESS;
7437}
7438
7439
7440/**
7441 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7442 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7443 *
7444 * @param pVCpu The cross context virtual CPU structure.
7445 */
7446DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7447{
7448 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7449 {
7450 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7451 {
7452 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7453 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7454 AssertRC(rc);
7455 Log4(("Setup interrupt-window exiting\n"));
7456 }
7457 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7458}
7459
7460
7461/**
7462 * Clears the interrupt-window exiting control in the VMCS.
7463 *
7464 * @param pVCpu The cross context virtual CPU structure.
7465 */
7466DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7467{
7468 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7469 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7470 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7471 AssertRC(rc);
7472 Log4(("Cleared interrupt-window exiting\n"));
7473}
7474
7475
7476/**
7477 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7478 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7479 *
7480 * @param pVCpu The cross context virtual CPU structure.
7481 */
7482DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7483{
7484 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7485 {
7486 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7487 {
7488 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7489 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7490 AssertRC(rc);
7491 Log4(("Setup NMI-window exiting\n"));
7492 }
7493 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7494}
7495
7496
7497/**
7498 * Clears the NMI-window exiting control in the VMCS.
7499 *
7500 * @param pVCpu The cross context virtual CPU structure.
7501 */
7502DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7503{
7504 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7505 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7506 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7507 AssertRC(rc);
7508 Log4(("Cleared NMI-window exiting\n"));
7509}
7510
7511
7512/**
7513 * Evaluates the event to be delivered to the guest and sets it as the pending
7514 * event.
7515 *
7516 * @returns The VT-x guest-interruptibility state.
7517 * @param pVCpu The cross context virtual CPU structure.
7518 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7519 * out-of-sync. Make sure to update the required fields
7520 * before using them.
7521 */
7522static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7523{
7524 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7525 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7526 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7527 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7528 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7529
7530 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7531 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7532 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7533 Assert(!TRPMHasTrap(pVCpu));
7534
7535#ifdef VBOX_WITH_NEW_APIC
7536 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7537 APICUpdatePendingInterrupts(pVCpu);
7538#endif
7539
7540 /*
7541 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7542 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7543 */
7544 /** @todo SMI. SMIs take priority over NMIs. */
7545 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7546 {
7547 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7548 if ( !pVCpu->hm.s.Event.fPending
7549 && !fBlockNmi
7550 && !fBlockSti
7551 && !fBlockMovSS)
7552 {
7553 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7554 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7555 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7556
7557 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7558 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7559 }
7560 else
7561 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7562 }
7563 /*
7564 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7565 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7566 */
7567 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7568 && !pVCpu->hm.s.fSingleInstruction)
7569 {
7570 Assert(!DBGFIsStepping(pVCpu));
7571 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7572 AssertRC(rc);
7573 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7574 if ( !pVCpu->hm.s.Event.fPending
7575 && !fBlockInt
7576 && !fBlockSti
7577 && !fBlockMovSS)
7578 {
7579 uint8_t u8Interrupt;
7580 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7581 if (RT_SUCCESS(rc))
7582 {
7583 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7584 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7585 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7586
7587 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7588 }
7589 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7590 {
7591 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7592 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7593 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7594 }
7595 else
7596 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7597 }
7598 else
7599 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7600 }
7601
7602 return uIntrState;
7603}
7604
7605
7606/**
7607 * Sets a pending-debug exception to be delivered to the guest if the guest is
7608 * single-stepping in the VMCS.
7609 *
7610 * @param pVCpu The cross context virtual CPU structure.
7611 */
7612DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7613{
7614 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7615 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7616 AssertRC(rc);
7617}
7618
7619
7620/**
7621 * Injects any pending events into the guest if the guest is in a state to
7622 * receive them.
7623 *
7624 * @returns Strict VBox status code (i.e. informational status codes too).
7625 * @param pVCpu The cross context virtual CPU structure.
7626 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7627 * out-of-sync. Make sure to update the required fields
7628 * before using them.
7629 * @param uIntrState The VT-x guest-interruptibility state.
7630 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7631 * return VINF_EM_DBG_STEPPED if the event was
7632 * dispatched directly.
7633 */
7634static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7635{
7636 HMVMX_ASSERT_PREEMPT_SAFE();
7637 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7638
7639 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7640 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7641
7642 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7643 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7644 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7645 Assert(!TRPMHasTrap(pVCpu));
7646
7647 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7648 if (pVCpu->hm.s.Event.fPending)
7649 {
7650 /*
7651 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7652 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7653 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7654 *
7655 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7656 */
7657 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7658#ifdef VBOX_STRICT
7659 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7660 {
7661 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7662 Assert(!fBlockInt);
7663 Assert(!fBlockSti);
7664 Assert(!fBlockMovSS);
7665 }
7666 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7667 {
7668 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7669 Assert(!fBlockSti);
7670 Assert(!fBlockMovSS);
7671 Assert(!fBlockNmi);
7672 }
7673#endif
7674 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7675 (uint8_t)uIntType));
7676 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7677 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7678 fStepping, &uIntrState);
7679 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7680
7681 /* Update the interruptibility-state as it could have been changed by
7682 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7683 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7684 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7685
7686 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7687 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7688 else
7689 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7690 }
7691
7692 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7693 if ( fBlockSti
7694 || fBlockMovSS)
7695 {
7696 if (!pVCpu->hm.s.fSingleInstruction)
7697 {
7698 /*
7699 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7700 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7701 * See Intel spec. 27.3.4 "Saving Non-Register State".
7702 */
7703 Assert(!DBGFIsStepping(pVCpu));
7704 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7705 AssertRCReturn(rc2, rc2);
7706 if (pMixedCtx->eflags.Bits.u1TF)
7707 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7708 }
7709 else if (pMixedCtx->eflags.Bits.u1TF)
7710 {
7711 /*
7712 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7713 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7714 */
7715 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7716 uIntrState = 0;
7717 }
7718 }
7719
7720 /*
7721 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7722 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7723 */
7724 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7725 AssertRC(rc2);
7726
7727 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7728 NOREF(fBlockMovSS); NOREF(fBlockSti);
7729 return rcStrict;
7730}
7731
7732
7733/**
7734 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7735 *
7736 * @param pVCpu The cross context virtual CPU structure.
7737 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7738 * out-of-sync. Make sure to update the required fields
7739 * before using them.
7740 */
7741DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7742{
7743 NOREF(pMixedCtx);
7744 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7745 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7746}
7747
7748
7749/**
7750 * Injects a double-fault (\#DF) exception into the VM.
7751 *
7752 * @returns Strict VBox status code (i.e. informational status codes too).
7753 * @param pVCpu The cross context virtual CPU structure.
7754 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7755 * out-of-sync. Make sure to update the required fields
7756 * before using them.
7757 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7758 * and should return VINF_EM_DBG_STEPPED if the event
7759 * is injected directly (register modified by us, not
7760 * by hardware on VM-entry).
7761 * @param puIntrState Pointer to the current guest interruptibility-state.
7762 * This interruptibility-state will be updated if
7763 * necessary. This cannot not be NULL.
7764 */
7765DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7766{
7767 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7768 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7769 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7770 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7771 fStepping, puIntrState);
7772}
7773
7774
7775/**
7776 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7777 *
7778 * @param pVCpu The cross context virtual CPU structure.
7779 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7780 * out-of-sync. Make sure to update the required fields
7781 * before using them.
7782 */
7783DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7784{
7785 NOREF(pMixedCtx);
7786 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7787 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7788 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7789}
7790
7791
7792/**
7793 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7794 *
7795 * @param pVCpu The cross context virtual CPU structure.
7796 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7797 * out-of-sync. Make sure to update the required fields
7798 * before using them.
7799 * @param cbInstr The value of RIP that is to be pushed on the guest
7800 * stack.
7801 */
7802DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7803{
7804 NOREF(pMixedCtx);
7805 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7806 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7807 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7808}
7809
7810
7811/**
7812 * Injects a general-protection (\#GP) fault into the VM.
7813 *
7814 * @returns Strict VBox status code (i.e. informational status codes too).
7815 * @param pVCpu The cross context virtual CPU structure.
7816 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7817 * out-of-sync. Make sure to update the required fields
7818 * before using them.
7819 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7820 * mode, i.e. in real-mode it's not valid).
7821 * @param u32ErrorCode The error code associated with the \#GP.
7822 * @param fStepping Whether we're running in
7823 * hmR0VmxRunGuestCodeStep() and should return
7824 * VINF_EM_DBG_STEPPED if the event is injected
7825 * directly (register modified by us, not by
7826 * hardware on VM-entry).
7827 * @param puIntrState Pointer to the current guest interruptibility-state.
7828 * This interruptibility-state will be updated if
7829 * necessary. This cannot not be NULL.
7830 */
7831DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7832 bool fStepping, uint32_t *puIntrState)
7833{
7834 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7835 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7836 if (fErrorCodeValid)
7837 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7838 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7839 fStepping, puIntrState);
7840}
7841
7842
7843/**
7844 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7845 * VM.
7846 *
7847 * @param pVCpu The cross context virtual CPU structure.
7848 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7849 * out-of-sync. Make sure to update the required fields
7850 * before using them.
7851 * @param u32ErrorCode The error code associated with the \#GP.
7852 */
7853DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7854{
7855 NOREF(pMixedCtx);
7856 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7857 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7858 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7859 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7860}
7861
7862
7863/**
7864 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7865 *
7866 * @param pVCpu The cross context virtual CPU structure.
7867 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7868 * out-of-sync. Make sure to update the required fields
7869 * before using them.
7870 * @param uVector The software interrupt vector number.
7871 * @param cbInstr The value of RIP that is to be pushed on the guest
7872 * stack.
7873 */
7874DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7875{
7876 NOREF(pMixedCtx);
7877 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7878 if ( uVector == X86_XCPT_BP
7879 || uVector == X86_XCPT_OF)
7880 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7881 else
7882 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7883 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7884}
7885
7886
7887/**
7888 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7889 * stack.
7890 *
7891 * @returns Strict VBox status code (i.e. informational status codes too).
7892 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7893 * @param pVM The cross context VM structure.
7894 * @param pMixedCtx Pointer to the guest-CPU context.
7895 * @param uValue The value to push to the guest stack.
7896 */
7897DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7898{
7899 /*
7900 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7901 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7902 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7903 */
7904 if (pMixedCtx->sp == 1)
7905 return VINF_EM_RESET;
7906 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7907 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7908 AssertRC(rc);
7909 return rc;
7910}
7911
7912
7913/**
7914 * Injects an event into the guest upon VM-entry by updating the relevant fields
7915 * in the VM-entry area in the VMCS.
7916 *
7917 * @returns Strict VBox status code (i.e. informational status codes too).
7918 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7919 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7920 *
7921 * @param pVCpu The cross context virtual CPU structure.
7922 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7923 * be out-of-sync. Make sure to update the required
7924 * fields before using them.
7925 * @param u64IntInfo The VM-entry interruption-information field.
7926 * @param cbInstr The VM-entry instruction length in bytes (for
7927 * software interrupts, exceptions and privileged
7928 * software exceptions).
7929 * @param u32ErrCode The VM-entry exception error code.
7930 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7931 * @param puIntrState Pointer to the current guest interruptibility-state.
7932 * This interruptibility-state will be updated if
7933 * necessary. This cannot not be NULL.
7934 * @param fStepping Whether we're running in
7935 * hmR0VmxRunGuestCodeStep() and should return
7936 * VINF_EM_DBG_STEPPED if the event is injected
7937 * directly (register modified by us, not by
7938 * hardware on VM-entry).
7939 *
7940 * @remarks Requires CR0!
7941 * @remarks No-long-jump zone!!!
7942 */
7943static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7944 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7945 uint32_t *puIntrState)
7946{
7947 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7948 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7949 Assert(puIntrState);
7950 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7951
7952 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7953 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7954
7955#ifdef VBOX_STRICT
7956 /* Validate the error-code-valid bit for hardware exceptions. */
7957 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7958 {
7959 switch (uVector)
7960 {
7961 case X86_XCPT_PF:
7962 case X86_XCPT_DF:
7963 case X86_XCPT_TS:
7964 case X86_XCPT_NP:
7965 case X86_XCPT_SS:
7966 case X86_XCPT_GP:
7967 case X86_XCPT_AC:
7968 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7969 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7970 /* fallthru */
7971 default:
7972 break;
7973 }
7974 }
7975#endif
7976
7977 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7978 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7979 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7980
7981 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7982
7983 /* We require CR0 to check if the guest is in real-mode. */
7984 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7985 AssertRCReturn(rc, rc);
7986
7987 /*
7988 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7989 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7990 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7991 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7992 */
7993 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7994 {
7995 PVM pVM = pVCpu->CTX_SUFF(pVM);
7996 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7997 {
7998 Assert(PDMVmmDevHeapIsEnabled(pVM));
7999 Assert(pVM->hm.s.vmx.pRealModeTSS);
8000
8001 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8002 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8003 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8004 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8005 AssertRCReturn(rc, rc);
8006 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8007
8008 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8009 size_t const cbIdtEntry = sizeof(X86IDTR16);
8010 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8011 {
8012 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8013 if (uVector == X86_XCPT_DF)
8014 return VINF_EM_RESET;
8015
8016 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8017 if (uVector == X86_XCPT_GP)
8018 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8019
8020 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8021 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8022 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8023 fStepping, puIntrState);
8024 }
8025
8026 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8027 uint16_t uGuestIp = pMixedCtx->ip;
8028 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8029 {
8030 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8031 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8032 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8033 }
8034 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8035 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8036
8037 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8038 X86IDTR16 IdtEntry;
8039 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8040 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8041 AssertRCReturn(rc, rc);
8042
8043 /* Construct the stack frame for the interrupt/exception handler. */
8044 VBOXSTRICTRC rcStrict;
8045 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8046 if (rcStrict == VINF_SUCCESS)
8047 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8048 if (rcStrict == VINF_SUCCESS)
8049 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8050
8051 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8052 if (rcStrict == VINF_SUCCESS)
8053 {
8054 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8055 pMixedCtx->rip = IdtEntry.offSel;
8056 pMixedCtx->cs.Sel = IdtEntry.uSel;
8057 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8058 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8059 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8060 && uVector == X86_XCPT_PF)
8061 pMixedCtx->cr2 = GCPtrFaultAddress;
8062
8063 /* If any other guest-state bits are changed here, make sure to update
8064 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8065 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8066 | HM_CHANGED_GUEST_RIP
8067 | HM_CHANGED_GUEST_RFLAGS
8068 | HM_CHANGED_GUEST_RSP);
8069
8070 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8071 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8072 {
8073 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8074 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8075 Log4(("Clearing inhibition due to STI.\n"));
8076 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8077 }
8078 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8079 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8080
8081 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8082 it, if we are returning to ring-3 before executing guest code. */
8083 pVCpu->hm.s.Event.fPending = false;
8084
8085 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8086 if (fStepping)
8087 rcStrict = VINF_EM_DBG_STEPPED;
8088 }
8089 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8090 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8091 return rcStrict;
8092 }
8093
8094 /*
8095 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8096 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8097 */
8098 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8099 }
8100
8101 /* Validate. */
8102 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8103 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8104 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8105
8106 /* Inject. */
8107 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8108 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8109 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8110 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8111
8112 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8113 && uVector == X86_XCPT_PF)
8114 pMixedCtx->cr2 = GCPtrFaultAddress;
8115
8116 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8117 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8118
8119 AssertRCReturn(rc, rc);
8120 return VINF_SUCCESS;
8121}
8122
8123
8124/**
8125 * Clears the interrupt-window exiting control in the VMCS and if necessary
8126 * clears the current event in the VMCS as well.
8127 *
8128 * @returns VBox status code.
8129 * @param pVCpu The cross context virtual CPU structure.
8130 *
8131 * @remarks Use this function only to clear events that have not yet been
8132 * delivered to the guest but are injected in the VMCS!
8133 * @remarks No-long-jump zone!!!
8134 */
8135static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8136{
8137 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8138
8139 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8140 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8141
8142 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8143 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8144}
8145
8146
8147/**
8148 * Enters the VT-x session.
8149 *
8150 * @returns VBox status code.
8151 * @param pVM The cross context VM structure.
8152 * @param pVCpu The cross context virtual CPU structure.
8153 * @param pCpu Pointer to the CPU info struct.
8154 */
8155VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8156{
8157 AssertPtr(pVM);
8158 AssertPtr(pVCpu);
8159 Assert(pVM->hm.s.vmx.fSupported);
8160 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8161 NOREF(pCpu); NOREF(pVM);
8162
8163 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8164 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8165
8166#ifdef VBOX_STRICT
8167 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8168 RTCCUINTREG uHostCR4 = ASMGetCR4();
8169 if (!(uHostCR4 & X86_CR4_VMXE))
8170 {
8171 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8172 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8173 }
8174#endif
8175
8176 /*
8177 * Load the VCPU's VMCS as the current (and active) one.
8178 */
8179 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8180 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8181 if (RT_FAILURE(rc))
8182 return rc;
8183
8184 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8185 pVCpu->hm.s.fLeaveDone = false;
8186 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8187
8188 return VINF_SUCCESS;
8189}
8190
8191
8192/**
8193 * The thread-context callback (only on platforms which support it).
8194 *
8195 * @param enmEvent The thread-context event.
8196 * @param pVCpu The cross context virtual CPU structure.
8197 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8198 * @thread EMT(pVCpu)
8199 */
8200VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8201{
8202 NOREF(fGlobalInit);
8203
8204 switch (enmEvent)
8205 {
8206 case RTTHREADCTXEVENT_OUT:
8207 {
8208 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8209 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8210 VMCPU_ASSERT_EMT(pVCpu);
8211
8212 PVM pVM = pVCpu->CTX_SUFF(pVM);
8213 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8214
8215 /* No longjmps (logger flushes, locks) in this fragile context. */
8216 VMMRZCallRing3Disable(pVCpu);
8217 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8218
8219 /*
8220 * Restore host-state (FPU, debug etc.)
8221 */
8222 if (!pVCpu->hm.s.fLeaveDone)
8223 {
8224 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8225 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8226 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8227 pVCpu->hm.s.fLeaveDone = true;
8228 }
8229
8230 /* Leave HM context, takes care of local init (term). */
8231 int rc = HMR0LeaveCpu(pVCpu);
8232 AssertRC(rc); NOREF(rc);
8233
8234 /* Restore longjmp state. */
8235 VMMRZCallRing3Enable(pVCpu);
8236 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8237 break;
8238 }
8239
8240 case RTTHREADCTXEVENT_IN:
8241 {
8242 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8243 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8244 VMCPU_ASSERT_EMT(pVCpu);
8245
8246 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8247 VMMRZCallRing3Disable(pVCpu);
8248 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8249
8250 /* Initialize the bare minimum state required for HM. This takes care of
8251 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8252 int rc = HMR0EnterCpu(pVCpu);
8253 AssertRC(rc);
8254 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8255
8256 /* Load the active VMCS as the current one. */
8257 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8258 {
8259 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8260 AssertRC(rc); NOREF(rc);
8261 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8262 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8263 }
8264 pVCpu->hm.s.fLeaveDone = false;
8265
8266 /* Restore longjmp state. */
8267 VMMRZCallRing3Enable(pVCpu);
8268 break;
8269 }
8270
8271 default:
8272 break;
8273 }
8274}
8275
8276
8277/**
8278 * Saves the host state in the VMCS host-state.
8279 * Sets up the VM-exit MSR-load area.
8280 *
8281 * The CPU state will be loaded from these fields on every successful VM-exit.
8282 *
8283 * @returns VBox status code.
8284 * @param pVM The cross context VM structure.
8285 * @param pVCpu The cross context virtual CPU structure.
8286 *
8287 * @remarks No-long-jump zone!!!
8288 */
8289static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8290{
8291 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8292
8293 int rc = VINF_SUCCESS;
8294 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8295 {
8296 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8297 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8298
8299 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8300 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8301
8302 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8303 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8304
8305 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8306 }
8307 return rc;
8308}
8309
8310
8311/**
8312 * Saves the host state in the VMCS host-state.
8313 *
8314 * @returns VBox status code.
8315 * @param pVM The cross context VM structure.
8316 * @param pVCpu The cross context virtual CPU structure.
8317 *
8318 * @remarks No-long-jump zone!!!
8319 */
8320VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8321{
8322 AssertPtr(pVM);
8323 AssertPtr(pVCpu);
8324
8325 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8326
8327 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8328 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8329 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8330 return hmR0VmxSaveHostState(pVM, pVCpu);
8331}
8332
8333
8334/**
8335 * Loads the guest state into the VMCS guest-state area.
8336 *
8337 * The will typically be done before VM-entry when the guest-CPU state and the
8338 * VMCS state may potentially be out of sync.
8339 *
8340 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8341 * VM-entry controls.
8342 * Sets up the appropriate VMX non-root function to execute guest code based on
8343 * the guest CPU mode.
8344 *
8345 * @returns VBox strict status code.
8346 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8347 * without unrestricted guest access and the VMMDev is not presently
8348 * mapped (e.g. EFI32).
8349 *
8350 * @param pVM The cross context VM structure.
8351 * @param pVCpu The cross context virtual CPU structure.
8352 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8353 * out-of-sync. Make sure to update the required fields
8354 * before using them.
8355 *
8356 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8357 * caller disables then again on successfull return. Confusing.)
8358 */
8359static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8360{
8361 AssertPtr(pVM);
8362 AssertPtr(pVCpu);
8363 AssertPtr(pMixedCtx);
8364 HMVMX_ASSERT_PREEMPT_SAFE();
8365
8366 VMMRZCallRing3Disable(pVCpu);
8367 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8368
8369 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8370
8371 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8372
8373 /* Determine real-on-v86 mode. */
8374 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8375 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8376 && CPUMIsGuestInRealModeEx(pMixedCtx))
8377 {
8378 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8379 }
8380
8381 /*
8382 * Load the guest-state into the VMCS.
8383 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8384 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8385 */
8386 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8387 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8388
8389 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8390 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8391 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8392
8393 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8394 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8395 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8396
8397 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8398 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8399
8400 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8401 if (rcStrict == VINF_SUCCESS)
8402 { /* likely */ }
8403 else
8404 {
8405 VMMRZCallRing3Enable(pVCpu);
8406 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8407 return rcStrict;
8408 }
8409
8410 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8411 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8412 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8413
8414 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8415 determine we don't have to swap EFER after all. */
8416 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8417 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8418
8419 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8420 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8421
8422 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8423 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8424
8425 /*
8426 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8427 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8428 */
8429 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8430 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8431
8432 /* Clear any unused and reserved bits. */
8433 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8434
8435 VMMRZCallRing3Enable(pVCpu);
8436
8437 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8438 return rc;
8439}
8440
8441
8442/**
8443 * Loads the state shared between the host and guest into the VMCS.
8444 *
8445 * @param pVM The cross context VM structure.
8446 * @param pVCpu The cross context virtual CPU structure.
8447 * @param pCtx Pointer to the guest-CPU context.
8448 *
8449 * @remarks No-long-jump zone!!!
8450 */
8451static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8452{
8453 NOREF(pVM);
8454
8455 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8456 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8457
8458 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8459 {
8460 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8461 AssertRC(rc);
8462 }
8463
8464 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8465 {
8466 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8467 AssertRC(rc);
8468
8469 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8470 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8471 {
8472 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8473 AssertRC(rc);
8474 }
8475 }
8476
8477 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8478 {
8479 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8480 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8481 }
8482
8483 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8484 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8485 {
8486 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8487 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8488 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8489 AssertRC(rc);
8490 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8491 }
8492
8493 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8494 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8495}
8496
8497
8498/**
8499 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8500 *
8501 * @returns Strict VBox status code (i.e. informational status codes too).
8502 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8503 * without unrestricted guest access and the VMMDev is not presently
8504 * mapped (e.g. EFI32).
8505 *
8506 * @param pVM The cross context VM structure.
8507 * @param pVCpu The cross context virtual CPU structure.
8508 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8509 * out-of-sync. Make sure to update the required fields
8510 * before using them.
8511 */
8512static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8513{
8514 HMVMX_ASSERT_PREEMPT_SAFE();
8515
8516 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8517#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8518 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8519#endif
8520
8521 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8522 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8523 {
8524 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8525 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8526 { /* likely */}
8527 else
8528 {
8529 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8530 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8531 }
8532 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8533 }
8534 else if (HMCPU_CF_VALUE(pVCpu))
8535 {
8536 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8537 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8538 { /* likely */}
8539 else
8540 {
8541 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8542 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8543 return rcStrict;
8544 }
8545 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8546 }
8547
8548 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8549 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8550 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8551 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8552 return rcStrict;
8553}
8554
8555
8556/**
8557 * Does the preparations before executing guest code in VT-x.
8558 *
8559 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8560 * recompiler/IEM. We must be cautious what we do here regarding committing
8561 * guest-state information into the VMCS assuming we assuredly execute the
8562 * guest in VT-x mode.
8563 *
8564 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8565 * the common-state (TRPM/forceflags), we must undo those changes so that the
8566 * recompiler/IEM can (and should) use them when it resumes guest execution.
8567 * Otherwise such operations must be done when we can no longer exit to ring-3.
8568 *
8569 * @returns Strict VBox status code (i.e. informational status codes too).
8570 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8571 * have been disabled.
8572 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8573 * double-fault into the guest.
8574 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8575 * dispatched directly.
8576 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8577 *
8578 * @param pVM The cross context VM structure.
8579 * @param pVCpu The cross context virtual CPU structure.
8580 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8581 * out-of-sync. Make sure to update the required fields
8582 * before using them.
8583 * @param pVmxTransient Pointer to the VMX transient structure.
8584 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8585 * us ignore some of the reasons for returning to
8586 * ring-3, and return VINF_EM_DBG_STEPPED if event
8587 * dispatching took place.
8588 */
8589static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8590{
8591 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8592
8593#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8594 PGMRZDynMapFlushAutoSet(pVCpu);
8595#endif
8596
8597 /* Check force flag actions that might require us to go back to ring-3. */
8598 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8599 if (rcStrict == VINF_SUCCESS)
8600 { /* FFs doesn't get set all the time. */ }
8601 else
8602 return rcStrict;
8603
8604#ifndef IEM_VERIFICATION_MODE_FULL
8605 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8606 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8607 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8608 {
8609 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8610 RTGCPHYS GCPhysApicBase;
8611 GCPhysApicBase = pMixedCtx->msrApicBase;
8612 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8613
8614 /* Unalias any existing mapping. */
8615 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8616 AssertRCReturn(rc, rc);
8617
8618 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8619 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8620 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8621 AssertRCReturn(rc, rc);
8622
8623 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8624 }
8625#endif /* !IEM_VERIFICATION_MODE_FULL */
8626
8627 if (TRPMHasTrap(pVCpu))
8628 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8629 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8630
8631 /*
8632 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8633 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8634 */
8635 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8636 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8637 { /* likely */ }
8638 else
8639 {
8640 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8641 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8642 return rcStrict;
8643 }
8644
8645 /*
8646 * Load the guest state bits, we can handle longjmps/getting preempted here.
8647 *
8648 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8649 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8650 * Hence, this needs to be done -after- injection of events.
8651 */
8652 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8653 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8654 { /* likely */ }
8655 else
8656 return rcStrict;
8657
8658 /*
8659 * No longjmps to ring-3 from this point on!!!
8660 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8661 * This also disables flushing of the R0-logger instance (if any).
8662 */
8663 VMMRZCallRing3Disable(pVCpu);
8664
8665 /*
8666 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8667 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8668 *
8669 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8670 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8671 *
8672 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8673 * executing guest code.
8674 */
8675 pVmxTransient->fEFlags = ASMIntDisableFlags();
8676
8677 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8678 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8679 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8680 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8681 {
8682 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8683 {
8684 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8685 pVCpu->hm.s.Event.fPending = false;
8686
8687 return VINF_SUCCESS;
8688 }
8689
8690 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8691 rcStrict = VINF_EM_RAW_INTERRUPT;
8692 }
8693 else
8694 {
8695 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8696 rcStrict = VINF_EM_RAW_TO_R3;
8697 }
8698
8699 ASMSetFlags(pVmxTransient->fEFlags);
8700 VMMRZCallRing3Enable(pVCpu);
8701
8702 return rcStrict;
8703}
8704
8705
8706/**
8707 * Prepares to run guest code in VT-x and we've committed to doing so. This
8708 * means there is no backing out to ring-3 or anywhere else at this
8709 * point.
8710 *
8711 * @param pVM The cross context VM structure.
8712 * @param pVCpu The cross context virtual CPU structure.
8713 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8714 * out-of-sync. Make sure to update the required fields
8715 * before using them.
8716 * @param pVmxTransient Pointer to the VMX transient structure.
8717 *
8718 * @remarks Called with preemption disabled.
8719 * @remarks No-long-jump zone!!!
8720 */
8721static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8722{
8723 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8724 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8725 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8726
8727 /*
8728 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8729 */
8730 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8731 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8732
8733#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8734 if (!CPUMIsGuestFPUStateActive(pVCpu))
8735 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8736 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8737 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8738#endif
8739
8740 if ( pVCpu->hm.s.fPreloadGuestFpu
8741 && !CPUMIsGuestFPUStateActive(pVCpu))
8742 {
8743 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8744 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8745 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8746 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8747 }
8748
8749 /*
8750 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8751 */
8752 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8753 && pVCpu->hm.s.vmx.cMsrs > 0)
8754 {
8755 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8756 }
8757
8758 /*
8759 * Load the host state bits as we may've been preempted (only happens when
8760 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8761 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8762 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8763 * See @bugref{8432}.
8764 */
8765 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8766 {
8767 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8768 AssertRC(rc);
8769 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8770 }
8771 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8772
8773 /*
8774 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8775 */
8776 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8777 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8778 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8779
8780 /* Store status of the shared guest-host state at the time of VM-entry. */
8781#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8782 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8783 {
8784 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8785 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8786 }
8787 else
8788#endif
8789 {
8790 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8791 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8792 }
8793 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8794
8795 /*
8796 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8797 */
8798 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8799 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8800
8801 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8802 RTCPUID idCurrentCpu = pCpu->idCpu;
8803 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8804 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8805 {
8806 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8807 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8808 }
8809
8810 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8811 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8812 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8813 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8814
8815 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8816
8817 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8818 to start executing. */
8819
8820 /*
8821 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8822 */
8823 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8824 {
8825 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8826 {
8827 bool fMsrUpdated;
8828 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8829 AssertRC(rc2);
8830 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8831
8832 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8833 &fMsrUpdated);
8834 AssertRC(rc2);
8835 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8836
8837 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8838 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8839 }
8840 else
8841 {
8842 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8843 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8844 }
8845 }
8846
8847#ifdef VBOX_STRICT
8848 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8849 hmR0VmxCheckHostEferMsr(pVCpu);
8850 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8851#endif
8852#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8853 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8854 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8855 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8856#endif
8857}
8858
8859
8860/**
8861 * Performs some essential restoration of state after running guest code in
8862 * VT-x.
8863 *
8864 * @param pVM The cross context VM structure.
8865 * @param pVCpu The cross context virtual CPU structure.
8866 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8867 * out-of-sync. Make sure to update the required fields
8868 * before using them.
8869 * @param pVmxTransient Pointer to the VMX transient structure.
8870 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8871 *
8872 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8873 *
8874 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8875 * unconditionally when it is safe to do so.
8876 */
8877static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8878{
8879 NOREF(pVM);
8880
8881 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8882
8883 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8884 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8885 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8886 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8887 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8888 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8889
8890 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8891 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8892
8893 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8894 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8895 Assert(!ASMIntAreEnabled());
8896 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8897
8898#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8899 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8900 {
8901 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8902 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8903 }
8904#endif
8905
8906#if HC_ARCH_BITS == 64
8907 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8908#endif
8909#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8910 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8911 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8912 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8913#else
8914 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8915#endif
8916#ifdef VBOX_STRICT
8917 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8918#endif
8919 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8920 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8921
8922 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8923 uint32_t uExitReason;
8924 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8925 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8926 AssertRC(rc);
8927 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8928 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8929
8930 /* Update the VM-exit history array. */
8931 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8932
8933 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8934 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8935 {
8936 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8937 pVmxTransient->fVMEntryFailed));
8938 return;
8939 }
8940
8941 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8942 {
8943 /** @todo We can optimize this by only syncing with our force-flags when
8944 * really needed and keeping the VMCS state as it is for most
8945 * VM-exits. */
8946 /* Update the guest interruptibility-state from the VMCS. */
8947 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8948
8949#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8950 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8951 AssertRC(rc);
8952#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8953 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8954 AssertRC(rc);
8955#endif
8956
8957 /*
8958 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8959 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8960 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8961 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8962 */
8963 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8964 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8965 {
8966 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8967 AssertRC(rc);
8968 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8969 }
8970 }
8971}
8972
8973
8974/**
8975 * Runs the guest code using VT-x the normal way.
8976 *
8977 * @returns VBox status code.
8978 * @param pVM The cross context VM structure.
8979 * @param pVCpu The cross context virtual CPU structure.
8980 * @param pCtx Pointer to the guest-CPU context.
8981 *
8982 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8983 */
8984static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8985{
8986 VMXTRANSIENT VmxTransient;
8987 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8988 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8989 uint32_t cLoops = 0;
8990
8991 for (;; cLoops++)
8992 {
8993 Assert(!HMR0SuspendPending());
8994 HMVMX_ASSERT_CPU_SAFE();
8995
8996 /* Preparatory work for running guest code, this may force us to return
8997 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8998 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8999 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9000 if (rcStrict != VINF_SUCCESS)
9001 break;
9002
9003 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9004 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9005 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9006
9007 /* Restore any residual host-state and save any bits shared between host
9008 and guest into the guest-CPU state. Re-enables interrupts! */
9009 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
9010
9011 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9012 if (RT_SUCCESS(rcRun))
9013 { /* very likely */ }
9014 else
9015 {
9016 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9017 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9018 return rcRun;
9019 }
9020
9021 /* Profile the VM-exit. */
9022 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9023 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9024 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9025 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9026 HMVMX_START_EXIT_DISPATCH_PROF();
9027
9028 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9029
9030 /* Handle the VM-exit. */
9031#ifdef HMVMX_USE_FUNCTION_TABLE
9032 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9033#else
9034 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9035#endif
9036 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9037 if (rcStrict == VINF_SUCCESS)
9038 {
9039 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9040 continue; /* likely */
9041 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9042 rcStrict = VINF_EM_RAW_INTERRUPT;
9043 }
9044 break;
9045 }
9046
9047 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9048 return rcStrict;
9049}
9050
9051
9052
9053/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9054 * probes.
9055 *
9056 * The following few functions and associated structure contains the bloat
9057 * necessary for providing detailed debug events and dtrace probes as well as
9058 * reliable host side single stepping. This works on the principle of
9059 * "subclassing" the normal execution loop and workers. We replace the loop
9060 * method completely and override selected helpers to add necessary adjustments
9061 * to their core operation.
9062 *
9063 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9064 * any performance for debug and analysis features.
9065 *
9066 * @{
9067 */
9068
9069typedef struct VMXRUNDBGSTATE
9070{
9071 /** The RIP we started executing at. This is for detecting that we stepped. */
9072 uint64_t uRipStart;
9073 /** The CS we started executing with. */
9074 uint16_t uCsStart;
9075
9076 /** Whether we've actually modified the 1st execution control field. */
9077 bool fModifiedProcCtls : 1;
9078 /** Whether we've actually modified the 2nd execution control field. */
9079 bool fModifiedProcCtls2 : 1;
9080 /** Whether we've actually modified the exception bitmap. */
9081 bool fModifiedXcptBitmap : 1;
9082
9083 /** We desire the modified the CR0 mask to be cleared. */
9084 bool fClearCr0Mask : 1;
9085 /** We desire the modified the CR4 mask to be cleared. */
9086 bool fClearCr4Mask : 1;
9087 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9088 uint32_t fCpe1Extra;
9089 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9090 uint32_t fCpe1Unwanted;
9091 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9092 uint32_t fCpe2Extra;
9093 /** Extra stuff we need in */
9094 uint32_t bmXcptExtra;
9095 /** The sequence number of the Dtrace provider settings the state was
9096 * configured against. */
9097 uint32_t uDtraceSettingsSeqNo;
9098 /** Exits to check (one bit per exit). */
9099 uint32_t bmExitsToCheck[3];
9100
9101 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9102 uint32_t fProcCtlsInitial;
9103 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9104 uint32_t fProcCtls2Initial;
9105 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9106 uint32_t bmXcptInitial;
9107} VMXRUNDBGSTATE;
9108AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9109typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9110
9111
9112/**
9113 * Initializes the VMXRUNDBGSTATE structure.
9114 *
9115 * @param pVCpu The cross context virtual CPU structure of the
9116 * calling EMT.
9117 * @param pCtx The CPU register context to go with @a pVCpu.
9118 * @param pDbgState The structure to initialize.
9119 */
9120DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9121{
9122 pDbgState->uRipStart = pCtx->rip;
9123 pDbgState->uCsStart = pCtx->cs.Sel;
9124
9125 pDbgState->fModifiedProcCtls = false;
9126 pDbgState->fModifiedProcCtls2 = false;
9127 pDbgState->fModifiedXcptBitmap = false;
9128 pDbgState->fClearCr0Mask = false;
9129 pDbgState->fClearCr4Mask = false;
9130 pDbgState->fCpe1Extra = 0;
9131 pDbgState->fCpe1Unwanted = 0;
9132 pDbgState->fCpe2Extra = 0;
9133 pDbgState->bmXcptExtra = 0;
9134 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9135 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9136 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9137}
9138
9139
9140/**
9141 * Updates the VMSC fields with changes requested by @a pDbgState.
9142 *
9143 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9144 * immediately before executing guest code, i.e. when interrupts are disabled.
9145 * We don't check status codes here as we cannot easily assert or return in the
9146 * latter case.
9147 *
9148 * @param pVCpu The cross context virtual CPU structure.
9149 * @param pDbgState The debug state.
9150 */
9151DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9152{
9153 /*
9154 * Ensure desired flags in VMCS control fields are set.
9155 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9156 *
9157 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9158 * there should be no stale data in pCtx at this point.
9159 */
9160 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9161 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9162 {
9163 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9164 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9165 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9166 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9167 pDbgState->fModifiedProcCtls = true;
9168 }
9169
9170 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9171 {
9172 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9173 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9174 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9175 pDbgState->fModifiedProcCtls2 = true;
9176 }
9177
9178 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9179 {
9180 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9181 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9182 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9183 pDbgState->fModifiedXcptBitmap = true;
9184 }
9185
9186 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9187 {
9188 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9189 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9190 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9191 }
9192
9193 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9194 {
9195 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9196 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9197 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9198 }
9199}
9200
9201
9202DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9203{
9204 /*
9205 * Restore exit control settings as we may not reenter this function the
9206 * next time around.
9207 */
9208 /* We reload the initial value, trigger what we can of recalculations the
9209 next time around. From the looks of things, that's all that's required atm. */
9210 if (pDbgState->fModifiedProcCtls)
9211 {
9212 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9213 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9214 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9215 AssertRCReturn(rc2, rc2);
9216 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9217 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9218 }
9219
9220 /* We're currently the only ones messing with this one, so just restore the
9221 cached value and reload the field. */
9222 if ( pDbgState->fModifiedProcCtls2
9223 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9224 {
9225 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9226 AssertRCReturn(rc2, rc2);
9227 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9228 }
9229
9230 /* If we've modified the exception bitmap, we restore it and trigger
9231 reloading and partial recalculation the next time around. */
9232 if (pDbgState->fModifiedXcptBitmap)
9233 {
9234 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9235 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9236 }
9237
9238 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9239 if (pDbgState->fClearCr0Mask)
9240 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9241
9242 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9243 if (pDbgState->fClearCr4Mask)
9244 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9245
9246 return rcStrict;
9247}
9248
9249
9250/**
9251 * Configures VM-exit controls for current DBGF and DTrace settings.
9252 *
9253 * This updates @a pDbgState and the VMCS execution control fields to reflect
9254 * the necessary exits demanded by DBGF and DTrace.
9255 *
9256 * @param pVM The cross context VM structure.
9257 * @param pVCpu The cross context virtual CPU structure.
9258 * @param pCtx Pointer to the guest-CPU context.
9259 * @param pDbgState The debug state.
9260 * @param pVmxTransient Pointer to the VMX transient structure. May update
9261 * fUpdateTscOffsettingAndPreemptTimer.
9262 */
9263static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9264 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9265{
9266 /*
9267 * Take down the dtrace serial number so we can spot changes.
9268 */
9269 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9270 ASMCompilerBarrier();
9271
9272 /*
9273 * We'll rebuild most of the middle block of data members (holding the
9274 * current settings) as we go along here, so start by clearing it all.
9275 */
9276 pDbgState->bmXcptExtra = 0;
9277 pDbgState->fCpe1Extra = 0;
9278 pDbgState->fCpe1Unwanted = 0;
9279 pDbgState->fCpe2Extra = 0;
9280 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9281 pDbgState->bmExitsToCheck[i] = 0;
9282
9283 /*
9284 * Software interrupts (INT XXh) - no idea how to trigger these...
9285 */
9286 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9287 || VBOXVMM_INT_SOFTWARE_ENABLED())
9288 {
9289 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9290 }
9291
9292 /*
9293 * Exception bitmap and XCPT events+probes.
9294 */
9295 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9296 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9297 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9298
9299 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9300 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9301 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9302 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9303 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9304 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9305 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9306 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9307 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9308 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9309 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9310 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9311 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9312 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9313 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9314 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9315 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9316 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9317
9318 if (pDbgState->bmXcptExtra)
9319 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9320
9321 /*
9322 * Process events and probes for VM exits, making sure we get the wanted exits.
9323 *
9324 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9325 * So, when adding/changing/removing please don't forget to update it.
9326 *
9327 * Some of the macros are picking up local variables to save horizontal space,
9328 * (being able to see it in a table is the lesser evil here).
9329 */
9330#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9331 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9332 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9333#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9334 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9335 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9336 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9337 } else do { } while (0)
9338#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9339 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9340 { \
9341 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9342 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9343 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9344 } else do { } while (0)
9345#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9346 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9347 { \
9348 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9349 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9350 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9351 } else do { } while (0)
9352#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9353 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9354 { \
9355 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9356 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9357 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9358 } else do { } while (0)
9359
9360 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9361 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9362 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9363 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9364 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9365
9366 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9367 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9368 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9369 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9370 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9371 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9372 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9373 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9374 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9375 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9376 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9377 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9378 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9379 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9380 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9381 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9382 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9383 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9384 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9385 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9386 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9387 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9388 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9389 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9390 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9391 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9392 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9393 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9394 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9395 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9396 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9397 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9398 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9399 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9400 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9402
9403 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9404 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9405 {
9406 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9407 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9408 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9409 AssertRC(rc2);
9410
9411#if 0 /** @todo fix me */
9412 pDbgState->fClearCr0Mask = true;
9413 pDbgState->fClearCr4Mask = true;
9414#endif
9415 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9416 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9417 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9418 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9419 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9420 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9421 require clearing here and in the loop if we start using it. */
9422 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9423 }
9424 else
9425 {
9426 if (pDbgState->fClearCr0Mask)
9427 {
9428 pDbgState->fClearCr0Mask = false;
9429 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9430 }
9431 if (pDbgState->fClearCr4Mask)
9432 {
9433 pDbgState->fClearCr4Mask = false;
9434 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9435 }
9436 }
9437 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9438 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9439
9440 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9441 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9442 {
9443 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9444 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9445 }
9446 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9447 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9448
9449 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9450 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9451 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9452 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9453 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9454 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9455 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9456 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9457#if 0 /** @todo too slow, fix handler. */
9458 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9459#endif
9460 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9461
9462 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9463 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9464 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9465 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9466 {
9467 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9468 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9469 }
9470 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9471 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9472 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9473 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9474
9475 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9476 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9477 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9478 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9479 {
9480 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9481 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9482 }
9483 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9484 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9485 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9486 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9487
9488 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9489 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9490 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9491 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9492 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9493 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9494 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9495 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9496 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9497 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9498 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9499 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9500 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9501 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9502 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9503 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9504 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9505 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9506 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9507 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9508 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9509 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9510
9511#undef IS_EITHER_ENABLED
9512#undef SET_ONLY_XBM_IF_EITHER_EN
9513#undef SET_CPE1_XBM_IF_EITHER_EN
9514#undef SET_CPEU_XBM_IF_EITHER_EN
9515#undef SET_CPE2_XBM_IF_EITHER_EN
9516
9517 /*
9518 * Sanitize the control stuff.
9519 */
9520 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9521 if (pDbgState->fCpe2Extra)
9522 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9523 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9524 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9525 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9526 {
9527 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9528 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9529 }
9530
9531 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9532 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9533 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9534 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9535}
9536
9537
9538/**
9539 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9540 *
9541 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9542 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9543 * either.
9544 *
9545 * @returns Strict VBox status code (i.e. informational status codes too).
9546 * @param pVM The cross context VM structure.
9547 * @param pVCpu The cross context virtual CPU structure.
9548 * @param pMixedCtx Pointer to the guest-CPU context.
9549 * @param pVmxTransient Pointer to the VMX-transient structure.
9550 * @param uExitReason The VM-exit reason.
9551 *
9552 * @remarks The name of this function is displayed by dtrace, so keep it short
9553 * and to the point. No longer than 33 chars long, please.
9554 */
9555static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9556 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9557{
9558 /*
9559 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9560 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9561 *
9562 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9563 * does. Must add/change/remove both places. Same ordering, please.
9564 *
9565 * Added/removed events must also be reflected in the next section
9566 * where we dispatch dtrace events.
9567 */
9568 bool fDtrace1 = false;
9569 bool fDtrace2 = false;
9570 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9571 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9572 uint32_t uEventArg = 0;
9573#define SET_EXIT(a_EventSubName) \
9574 do { \
9575 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9576 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9577 } while (0)
9578#define SET_BOTH(a_EventSubName) \
9579 do { \
9580 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9581 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9582 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9583 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9584 } while (0)
9585 switch (uExitReason)
9586 {
9587 case VMX_EXIT_MTF:
9588 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9589
9590 case VMX_EXIT_XCPT_OR_NMI:
9591 {
9592 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9593 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9594 {
9595 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9596 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9597 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9598 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9599 {
9600 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9601 {
9602 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9603 uEventArg = pVmxTransient->uExitIntErrorCode;
9604 }
9605 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9606 switch (enmEvent1)
9607 {
9608 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9609 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9610 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9611 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9612 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9613 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9614 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9615 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9616 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9617 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9618 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9619 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9620 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9621 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9622 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9623 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9624 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9625 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9626 default: break;
9627 }
9628 }
9629 else
9630 AssertFailed();
9631 break;
9632
9633 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9634 uEventArg = idxVector;
9635 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9636 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9637 break;
9638 }
9639 break;
9640 }
9641
9642 case VMX_EXIT_TRIPLE_FAULT:
9643 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9644 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9645 break;
9646 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9647 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9648 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9649 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9650 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9651
9652 /* Instruction specific VM-exits: */
9653 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9654 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9655 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9656 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9657 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9658 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9659 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9660 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9661 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9662 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9663 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9664 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9665 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9666 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9667 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9668 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9669 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9670 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9671 case VMX_EXIT_MOV_CRX:
9672 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9673/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9674* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9675 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9676 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9677 SET_BOTH(CRX_READ);
9678 else
9679 SET_BOTH(CRX_WRITE);
9680 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9681 break;
9682 case VMX_EXIT_MOV_DRX:
9683 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9684 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9685 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9686 SET_BOTH(DRX_READ);
9687 else
9688 SET_BOTH(DRX_WRITE);
9689 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9690 break;
9691 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9692 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9693 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9694 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9695 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9696 case VMX_EXIT_XDTR_ACCESS:
9697 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9698 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9699 {
9700 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9701 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9702 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9703 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9704 }
9705 break;
9706
9707 case VMX_EXIT_TR_ACCESS:
9708 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9709 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9710 {
9711 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9712 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9713 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9714 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9715 }
9716 break;
9717
9718 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9719 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9720 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9721 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9722 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9723 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9724 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9725 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9726 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9727 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9728 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9729
9730 /* Events that aren't relevant at this point. */
9731 case VMX_EXIT_EXT_INT:
9732 case VMX_EXIT_INT_WINDOW:
9733 case VMX_EXIT_NMI_WINDOW:
9734 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9735 case VMX_EXIT_PREEMPT_TIMER:
9736 case VMX_EXIT_IO_INSTR:
9737 break;
9738
9739 /* Errors and unexpected events. */
9740 case VMX_EXIT_INIT_SIGNAL:
9741 case VMX_EXIT_SIPI:
9742 case VMX_EXIT_IO_SMI:
9743 case VMX_EXIT_SMI:
9744 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9745 case VMX_EXIT_ERR_MSR_LOAD:
9746 case VMX_EXIT_ERR_MACHINE_CHECK:
9747 break;
9748
9749 default:
9750 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9751 break;
9752 }
9753#undef SET_BOTH
9754#undef SET_EXIT
9755
9756 /*
9757 * Dtrace tracepoints go first. We do them here at once so we don't
9758 * have to copy the guest state saving and stuff a few dozen times.
9759 * Down side is that we've got to repeat the switch, though this time
9760 * we use enmEvent since the probes are a subset of what DBGF does.
9761 */
9762 if (fDtrace1 || fDtrace2)
9763 {
9764 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9765 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9766 switch (enmEvent1)
9767 {
9768 /** @todo consider which extra parameters would be helpful for each probe. */
9769 case DBGFEVENT_END: break;
9770 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9771 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9772 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9773 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9774 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9775 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9776 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9777 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9778 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9779 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9780 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9781 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9782 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9783 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9784 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9785 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9786 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9787 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9788 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9789 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9790 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9791 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9792 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9793 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9794 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9795 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9796 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9797 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9798 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9799 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9800 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9801 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9802 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9803 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9804 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9805 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9806 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9807 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9808 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9809 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9810 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9811 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9812 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9813 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9814 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9815 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9816 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9817 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9818 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9819 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9820 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9821 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9822 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9823 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9824 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9825 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9826 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9827 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9828 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9829 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9830 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9831 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9832 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9833 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9834 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9835 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9836 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9837 }
9838 switch (enmEvent2)
9839 {
9840 /** @todo consider which extra parameters would be helpful for each probe. */
9841 case DBGFEVENT_END: break;
9842 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9843 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9844 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9845 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9846 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9847 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9848 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9849 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9850 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9851 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9852 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9853 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9854 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9855 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9856 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9857 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9858 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9859 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9860 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9861 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9862 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9863 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9864 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9865 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9866 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9867 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9868 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9869 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9870 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9871 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9872 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9873 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9874 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9875 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9876 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9877 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9878 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9879 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9880 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9881 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9882 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9883 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9884 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9885 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9886 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9887 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9888 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9889 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9890 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9891 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9892 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9893 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9894 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9895 }
9896 }
9897
9898 /*
9899 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9900 * the DBGF call will do a full check).
9901 *
9902 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9903 * Note! If we have to events, we prioritize the first, i.e. the instruction
9904 * one, in order to avoid event nesting.
9905 */
9906 if ( enmEvent1 != DBGFEVENT_END
9907 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9908 {
9909 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9910 if (rcStrict != VINF_SUCCESS)
9911 return rcStrict;
9912 }
9913 else if ( enmEvent2 != DBGFEVENT_END
9914 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9915 {
9916 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9917 if (rcStrict != VINF_SUCCESS)
9918 return rcStrict;
9919 }
9920
9921 return VINF_SUCCESS;
9922}
9923
9924
9925/**
9926 * Single-stepping VM-exit filtering.
9927 *
9928 * This is preprocessing the exits and deciding whether we've gotten far enough
9929 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9930 * performed.
9931 *
9932 * @returns Strict VBox status code (i.e. informational status codes too).
9933 * @param pVM The cross context VM structure.
9934 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9935 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9936 * out-of-sync. Make sure to update the required
9937 * fields before using them.
9938 * @param pVmxTransient Pointer to the VMX-transient structure.
9939 * @param uExitReason The VM-exit reason.
9940 * @param pDbgState The debug state.
9941 */
9942DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9943 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9944{
9945 /*
9946 * Expensive (saves context) generic dtrace exit probe.
9947 */
9948 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9949 { /* more likely */ }
9950 else
9951 {
9952 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9953 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9954 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9955 }
9956
9957 /*
9958 * Check for host NMI, just to get that out of the way.
9959 */
9960 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9961 { /* normally likely */ }
9962 else
9963 {
9964 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9965 AssertRCReturn(rc2, rc2);
9966 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9967 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9968 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9969 }
9970
9971 /*
9972 * Check for single stepping event if we're stepping.
9973 */
9974 if (pVCpu->hm.s.fSingleInstruction)
9975 {
9976 switch (uExitReason)
9977 {
9978 case VMX_EXIT_MTF:
9979 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9980
9981 /* Various events: */
9982 case VMX_EXIT_XCPT_OR_NMI:
9983 case VMX_EXIT_EXT_INT:
9984 case VMX_EXIT_TRIPLE_FAULT:
9985 case VMX_EXIT_INT_WINDOW:
9986 case VMX_EXIT_NMI_WINDOW:
9987 case VMX_EXIT_TASK_SWITCH:
9988 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9989 case VMX_EXIT_APIC_ACCESS:
9990 case VMX_EXIT_EPT_VIOLATION:
9991 case VMX_EXIT_EPT_MISCONFIG:
9992 case VMX_EXIT_PREEMPT_TIMER:
9993
9994 /* Instruction specific VM-exits: */
9995 case VMX_EXIT_CPUID:
9996 case VMX_EXIT_GETSEC:
9997 case VMX_EXIT_HLT:
9998 case VMX_EXIT_INVD:
9999 case VMX_EXIT_INVLPG:
10000 case VMX_EXIT_RDPMC:
10001 case VMX_EXIT_RDTSC:
10002 case VMX_EXIT_RSM:
10003 case VMX_EXIT_VMCALL:
10004 case VMX_EXIT_VMCLEAR:
10005 case VMX_EXIT_VMLAUNCH:
10006 case VMX_EXIT_VMPTRLD:
10007 case VMX_EXIT_VMPTRST:
10008 case VMX_EXIT_VMREAD:
10009 case VMX_EXIT_VMRESUME:
10010 case VMX_EXIT_VMWRITE:
10011 case VMX_EXIT_VMXOFF:
10012 case VMX_EXIT_VMXON:
10013 case VMX_EXIT_MOV_CRX:
10014 case VMX_EXIT_MOV_DRX:
10015 case VMX_EXIT_IO_INSTR:
10016 case VMX_EXIT_RDMSR:
10017 case VMX_EXIT_WRMSR:
10018 case VMX_EXIT_MWAIT:
10019 case VMX_EXIT_MONITOR:
10020 case VMX_EXIT_PAUSE:
10021 case VMX_EXIT_XDTR_ACCESS:
10022 case VMX_EXIT_TR_ACCESS:
10023 case VMX_EXIT_INVEPT:
10024 case VMX_EXIT_RDTSCP:
10025 case VMX_EXIT_INVVPID:
10026 case VMX_EXIT_WBINVD:
10027 case VMX_EXIT_XSETBV:
10028 case VMX_EXIT_RDRAND:
10029 case VMX_EXIT_INVPCID:
10030 case VMX_EXIT_VMFUNC:
10031 case VMX_EXIT_RDSEED:
10032 case VMX_EXIT_XSAVES:
10033 case VMX_EXIT_XRSTORS:
10034 {
10035 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10036 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10037 AssertRCReturn(rc2, rc2);
10038 if ( pMixedCtx->rip != pDbgState->uRipStart
10039 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10040 return VINF_EM_DBG_STEPPED;
10041 break;
10042 }
10043
10044 /* Errors and unexpected events: */
10045 case VMX_EXIT_INIT_SIGNAL:
10046 case VMX_EXIT_SIPI:
10047 case VMX_EXIT_IO_SMI:
10048 case VMX_EXIT_SMI:
10049 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10050 case VMX_EXIT_ERR_MSR_LOAD:
10051 case VMX_EXIT_ERR_MACHINE_CHECK:
10052 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10053 break;
10054
10055 default:
10056 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
10057 break;
10058 }
10059 }
10060
10061 /*
10062 * Check for debugger event breakpoints and dtrace probes.
10063 */
10064 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10065 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10066 {
10067 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10068 if (rcStrict != VINF_SUCCESS)
10069 return rcStrict;
10070 }
10071
10072 /*
10073 * Normal processing.
10074 */
10075#ifdef HMVMX_USE_FUNCTION_TABLE
10076 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10077#else
10078 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10079#endif
10080}
10081
10082
10083/**
10084 * Single steps guest code using VT-x.
10085 *
10086 * @returns Strict VBox status code (i.e. informational status codes too).
10087 * @param pVM The cross context VM structure.
10088 * @param pVCpu The cross context virtual CPU structure.
10089 * @param pCtx Pointer to the guest-CPU context.
10090 *
10091 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10092 */
10093static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10094{
10095 VMXTRANSIENT VmxTransient;
10096 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10097
10098 /* Set HMCPU indicators. */
10099 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10100 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10101 pVCpu->hm.s.fDebugWantRdTscExit = false;
10102 pVCpu->hm.s.fUsingDebugLoop = true;
10103
10104 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10105 VMXRUNDBGSTATE DbgState;
10106 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10107 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10108
10109 /*
10110 * The loop.
10111 */
10112 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10113 for (uint32_t cLoops = 0; ; cLoops++)
10114 {
10115 Assert(!HMR0SuspendPending());
10116 HMVMX_ASSERT_CPU_SAFE();
10117 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10118
10119 /*
10120 * Preparatory work for running guest code, this may force us to return
10121 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10122 */
10123 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10124 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10125 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10126 if (rcStrict != VINF_SUCCESS)
10127 break;
10128
10129 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10130 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10131
10132 /*
10133 * Now we can run the guest code.
10134 */
10135 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10136
10137 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10138
10139 /*
10140 * Restore any residual host-state and save any bits shared between host
10141 * and guest into the guest-CPU state. Re-enables interrupts!
10142 */
10143 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
10144
10145 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10146 if (RT_SUCCESS(rcRun))
10147 { /* very likely */ }
10148 else
10149 {
10150 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10151 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10152 return rcRun;
10153 }
10154
10155 /* Profile the VM-exit. */
10156 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10157 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10158 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10159 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10160 HMVMX_START_EXIT_DISPATCH_PROF();
10161
10162 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10163
10164 /*
10165 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10166 */
10167 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10168 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10169 if (rcStrict != VINF_SUCCESS)
10170 break;
10171 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10172 {
10173 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10174 rcStrict = VINF_EM_RAW_INTERRUPT;
10175 break;
10176 }
10177
10178 /*
10179 * Stepping: Did the RIP change, if so, consider it a single step.
10180 * Otherwise, make sure one of the TFs gets set.
10181 */
10182 if (fStepping)
10183 {
10184 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10185 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10186 AssertRCReturn(rc2, rc2);
10187 if ( pCtx->rip != DbgState.uRipStart
10188 || pCtx->cs.Sel != DbgState.uCsStart)
10189 {
10190 rcStrict = VINF_EM_DBG_STEPPED;
10191 break;
10192 }
10193 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10194 }
10195
10196 /*
10197 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10198 */
10199 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10200 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10201 }
10202
10203 /*
10204 * Clear the X86_EFL_TF if necessary.
10205 */
10206 if (pVCpu->hm.s.fClearTrapFlag)
10207 {
10208 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10209 AssertRCReturn(rc2, rc2);
10210 pVCpu->hm.s.fClearTrapFlag = false;
10211 pCtx->eflags.Bits.u1TF = 0;
10212 }
10213 /** @todo there seems to be issues with the resume flag when the monitor trap
10214 * flag is pending without being used. Seen early in bios init when
10215 * accessing APIC page in protected mode. */
10216
10217 /*
10218 * Restore VM-exit control settings as we may not reenter this function the
10219 * next time around.
10220 */
10221 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10222
10223 /* Restore HMCPU indicators. */
10224 pVCpu->hm.s.fUsingDebugLoop = false;
10225 pVCpu->hm.s.fDebugWantRdTscExit = false;
10226 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10227
10228 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10229 return rcStrict;
10230}
10231
10232
10233/** @} */
10234
10235
10236/**
10237 * Checks if any expensive dtrace probes are enabled and we should go to the
10238 * debug loop.
10239 *
10240 * @returns true if we should use debug loop, false if not.
10241 */
10242static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10243{
10244 /* It's probably faster to OR the raw 32-bit counter variables together.
10245 Since the variables are in an array and the probes are next to one
10246 another (more or less), we have good locality. So, better read
10247 eight-nine cache lines ever time and only have one conditional, than
10248 128+ conditionals, right? */
10249 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10250 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10251 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10252 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10253 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10254 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10255 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10256 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10257 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10258 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10259 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10260 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10261 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10262 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10263 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10264 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10265 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10266 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10267 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10268 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10269 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10270 ) != 0
10271 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10272 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10273 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10274 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10275 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10276 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10277 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10278 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10279 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10280 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10281 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10282 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10283 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10284 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10285 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10286 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10287 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10288 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10289 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10290 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10291 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10292 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10293 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10294 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10295 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10296 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10297 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10298 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10299 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10300 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10301 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10302 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10303 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10304 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10305 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10306 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10307 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10308 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10309 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10310 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10311 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10312 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10313 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10314 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10315 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10316 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10317 ) != 0
10318 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10319 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10320 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10321 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10322 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10323 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10324 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10325 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10326 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10327 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10328 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10329 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10330 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10331 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10332 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10333 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10334 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10335 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10336 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10337 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10338 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10339 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10340 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10341 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10342 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10343 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10344 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10345 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10346 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10347 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10348 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10349 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10350 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10351 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10352 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10353 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10354 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10355 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10356 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10357 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10358 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10359 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10360 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10361 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10362 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10363 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10364 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10365 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10366 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10367 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10368 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10369 ) != 0;
10370}
10371
10372
10373/**
10374 * Runs the guest code using VT-x.
10375 *
10376 * @returns Strict VBox status code (i.e. informational status codes too).
10377 * @param pVM The cross context VM structure.
10378 * @param pVCpu The cross context virtual CPU structure.
10379 * @param pCtx Pointer to the guest-CPU context.
10380 */
10381VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10382{
10383 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10384 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10385 HMVMX_ASSERT_PREEMPT_SAFE();
10386
10387 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10388
10389 VBOXSTRICTRC rcStrict;
10390 if ( !pVCpu->hm.s.fUseDebugLoop
10391 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10392 && !DBGFIsStepping(pVCpu) )
10393 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10394 else
10395 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10396
10397 if (rcStrict == VERR_EM_INTERPRETER)
10398 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10399 else if (rcStrict == VINF_EM_RESET)
10400 rcStrict = VINF_EM_TRIPLE_FAULT;
10401
10402 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10403 if (RT_FAILURE(rc2))
10404 {
10405 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10406 rcStrict = rc2;
10407 }
10408 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10409 return rcStrict;
10410}
10411
10412
10413#ifndef HMVMX_USE_FUNCTION_TABLE
10414DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10415{
10416# ifdef DEBUG_ramshankar
10417# define RETURN_EXIT_CALL(a_CallExpr) \
10418 do { \
10419 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10420 VBOXSTRICTRC rcStrict = a_CallExpr; \
10421 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10422 return rcStrict; \
10423 } while (0)
10424# else
10425# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10426# endif
10427 switch (rcReason)
10428 {
10429 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10430 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10431 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10432 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10433 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10434 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10435 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10436 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10437 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10438 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10439 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10440 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10441 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10442 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10443 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10444 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10445 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10446 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10447 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10448 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10449 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10450 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10451 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10452 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10453 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10454 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10455 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10456 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10457 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10458 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10459 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10460 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10461 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10462 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10463
10464 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10465 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10466 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10467 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10468 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10469 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10470 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10471 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10472 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10473
10474 case VMX_EXIT_VMCLEAR:
10475 case VMX_EXIT_VMLAUNCH:
10476 case VMX_EXIT_VMPTRLD:
10477 case VMX_EXIT_VMPTRST:
10478 case VMX_EXIT_VMREAD:
10479 case VMX_EXIT_VMRESUME:
10480 case VMX_EXIT_VMWRITE:
10481 case VMX_EXIT_VMXOFF:
10482 case VMX_EXIT_VMXON:
10483 case VMX_EXIT_INVEPT:
10484 case VMX_EXIT_INVVPID:
10485 case VMX_EXIT_VMFUNC:
10486 case VMX_EXIT_XSAVES:
10487 case VMX_EXIT_XRSTORS:
10488 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10489 case VMX_EXIT_RESERVED_60:
10490 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10491 case VMX_EXIT_RESERVED_62:
10492 default:
10493 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10494 }
10495#undef RETURN_EXIT_CALL
10496}
10497#endif /* !HMVMX_USE_FUNCTION_TABLE */
10498
10499
10500#ifdef VBOX_STRICT
10501/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10502# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10503 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10504
10505# define HMVMX_ASSERT_PREEMPT_CPUID() \
10506 do { \
10507 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10508 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10509 } while (0)
10510
10511# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10512 do { \
10513 AssertPtr(pVCpu); \
10514 AssertPtr(pMixedCtx); \
10515 AssertPtr(pVmxTransient); \
10516 Assert(pVmxTransient->fVMEntryFailed == false); \
10517 Assert(ASMIntAreEnabled()); \
10518 HMVMX_ASSERT_PREEMPT_SAFE(); \
10519 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10520 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10521 HMVMX_ASSERT_PREEMPT_SAFE(); \
10522 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10523 HMVMX_ASSERT_PREEMPT_CPUID(); \
10524 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10525 } while (0)
10526
10527# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10528 do { \
10529 Log4Func(("\n")); \
10530 } while (0)
10531#else /* nonstrict builds: */
10532# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10533 do { \
10534 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10535 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10536 } while (0)
10537# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10538#endif
10539
10540
10541/**
10542 * Advances the guest RIP by the specified number of bytes.
10543 *
10544 * @param pVCpu The cross context virtual CPU structure.
10545 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10546 * out-of-sync. Make sure to update the required fields
10547 * before using them.
10548 * @param cbInstr Number of bytes to advance the RIP by.
10549 *
10550 * @remarks No-long-jump zone!!!
10551 */
10552DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10553{
10554 /* Advance the RIP. */
10555 pMixedCtx->rip += cbInstr;
10556 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10557
10558 /* Update interrupt inhibition. */
10559 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10560 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10561 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10562}
10563
10564
10565/**
10566 * Advances the guest RIP after reading it from the VMCS.
10567 *
10568 * @returns VBox status code, no informational status codes.
10569 * @param pVCpu The cross context virtual CPU structure.
10570 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10571 * out-of-sync. Make sure to update the required fields
10572 * before using them.
10573 * @param pVmxTransient Pointer to the VMX transient structure.
10574 *
10575 * @remarks No-long-jump zone!!!
10576 */
10577static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10578{
10579 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10580 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10581 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10582 AssertRCReturn(rc, rc);
10583
10584 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10585
10586 /*
10587 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10588 * pending debug exception field as it takes care of priority of events.
10589 *
10590 * See Intel spec. 32.2.1 "Debug Exceptions".
10591 */
10592 if ( !pVCpu->hm.s.fSingleInstruction
10593 && pMixedCtx->eflags.Bits.u1TF)
10594 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10595
10596 return VINF_SUCCESS;
10597}
10598
10599
10600/**
10601 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10602 * and update error record fields accordingly.
10603 *
10604 * @return VMX_IGS_* return codes.
10605 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10606 * wrong with the guest state.
10607 *
10608 * @param pVM The cross context VM structure.
10609 * @param pVCpu The cross context virtual CPU structure.
10610 * @param pCtx Pointer to the guest-CPU state.
10611 *
10612 * @remarks This function assumes our cache of the VMCS controls
10613 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10614 */
10615static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10616{
10617#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10618#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10619 uError = (err); \
10620 break; \
10621 } else do { } while (0)
10622
10623 int rc;
10624 uint32_t uError = VMX_IGS_ERROR;
10625 uint32_t u32Val;
10626 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10627
10628 do
10629 {
10630 /*
10631 * CR0.
10632 */
10633 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10634 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10635 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10636 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10637 if (fUnrestrictedGuest)
10638 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10639
10640 uint32_t u32GuestCR0;
10641 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10642 AssertRCBreak(rc);
10643 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10644 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10645 if ( !fUnrestrictedGuest
10646 && (u32GuestCR0 & X86_CR0_PG)
10647 && !(u32GuestCR0 & X86_CR0_PE))
10648 {
10649 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10650 }
10651
10652 /*
10653 * CR4.
10654 */
10655 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10656 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10657
10658 uint32_t u32GuestCR4;
10659 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10660 AssertRCBreak(rc);
10661 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10662 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10663
10664 /*
10665 * IA32_DEBUGCTL MSR.
10666 */
10667 uint64_t u64Val;
10668 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10669 AssertRCBreak(rc);
10670 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10671 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10672 {
10673 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10674 }
10675 uint64_t u64DebugCtlMsr = u64Val;
10676
10677#ifdef VBOX_STRICT
10678 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10679 AssertRCBreak(rc);
10680 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10681#endif
10682 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10683
10684 /*
10685 * RIP and RFLAGS.
10686 */
10687 uint32_t u32Eflags;
10688#if HC_ARCH_BITS == 64
10689 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10690 AssertRCBreak(rc);
10691 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10692 if ( !fLongModeGuest
10693 || !pCtx->cs.Attr.n.u1Long)
10694 {
10695 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10696 }
10697 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10698 * must be identical if the "IA-32e mode guest" VM-entry
10699 * control is 1 and CS.L is 1. No check applies if the
10700 * CPU supports 64 linear-address bits. */
10701
10702 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10703 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10704 AssertRCBreak(rc);
10705 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10706 VMX_IGS_RFLAGS_RESERVED);
10707 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10708 u32Eflags = u64Val;
10709#else
10710 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10711 AssertRCBreak(rc);
10712 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10713 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10714#endif
10715
10716 if ( fLongModeGuest
10717 || ( fUnrestrictedGuest
10718 && !(u32GuestCR0 & X86_CR0_PE)))
10719 {
10720 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10721 }
10722
10723 uint32_t u32EntryInfo;
10724 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10725 AssertRCBreak(rc);
10726 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10727 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10728 {
10729 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10730 }
10731
10732 /*
10733 * 64-bit checks.
10734 */
10735#if HC_ARCH_BITS == 64
10736 if (fLongModeGuest)
10737 {
10738 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10739 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10740 }
10741
10742 if ( !fLongModeGuest
10743 && (u32GuestCR4 & X86_CR4_PCIDE))
10744 {
10745 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10746 }
10747
10748 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10749 * 51:32 beyond the processor's physical-address width are 0. */
10750
10751 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10752 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10753 {
10754 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10755 }
10756
10757 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10758 AssertRCBreak(rc);
10759 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10760
10761 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10762 AssertRCBreak(rc);
10763 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10764#endif
10765
10766 /*
10767 * PERF_GLOBAL MSR.
10768 */
10769 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10770 {
10771 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10772 AssertRCBreak(rc);
10773 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10774 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10775 }
10776
10777 /*
10778 * PAT MSR.
10779 */
10780 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10781 {
10782 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10783 AssertRCBreak(rc);
10784 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10785 for (unsigned i = 0; i < 8; i++)
10786 {
10787 uint8_t u8Val = (u64Val & 0xff);
10788 if ( u8Val != 0 /* UC */
10789 && u8Val != 1 /* WC */
10790 && u8Val != 4 /* WT */
10791 && u8Val != 5 /* WP */
10792 && u8Val != 6 /* WB */
10793 && u8Val != 7 /* UC- */)
10794 {
10795 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10796 }
10797 u64Val >>= 8;
10798 }
10799 }
10800
10801 /*
10802 * EFER MSR.
10803 */
10804 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10805 {
10806 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10807 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10808 AssertRCBreak(rc);
10809 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10810 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10811 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10812 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10813 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10814 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10815 || !(u32GuestCR0 & X86_CR0_PG)
10816 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10817 VMX_IGS_EFER_LMA_LME_MISMATCH);
10818 }
10819
10820 /*
10821 * Segment registers.
10822 */
10823 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10824 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10825 if (!(u32Eflags & X86_EFL_VM))
10826 {
10827 /* CS */
10828 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10829 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10830 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10831 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10832 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10833 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10834 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10835 /* CS cannot be loaded with NULL in protected mode. */
10836 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10837 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10838 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10839 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10840 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10841 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10842 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10843 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10844 else
10845 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10846
10847 /* SS */
10848 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10849 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10850 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10851 if ( !(pCtx->cr0 & X86_CR0_PE)
10852 || pCtx->cs.Attr.n.u4Type == 3)
10853 {
10854 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10855 }
10856 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10857 {
10858 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10859 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10860 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10861 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10862 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10863 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10864 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10865 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10866 }
10867
10868 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10869 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10870 {
10871 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10872 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10873 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10874 || pCtx->ds.Attr.n.u4Type > 11
10875 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10876 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10877 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10878 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10879 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10880 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10881 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10882 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10883 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10884 }
10885 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10886 {
10887 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10888 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10889 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10890 || pCtx->es.Attr.n.u4Type > 11
10891 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10892 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10893 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10894 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10895 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10896 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10897 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10898 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10899 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10900 }
10901 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10902 {
10903 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10904 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10905 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10906 || pCtx->fs.Attr.n.u4Type > 11
10907 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10908 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10909 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10910 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10911 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10912 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10913 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10914 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10915 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10916 }
10917 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10918 {
10919 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10920 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10921 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10922 || pCtx->gs.Attr.n.u4Type > 11
10923 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10924 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10925 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10926 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10927 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10928 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10929 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10930 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10931 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10932 }
10933 /* 64-bit capable CPUs. */
10934#if HC_ARCH_BITS == 64
10935 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10936 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10937 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10938 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10939 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10940 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10941 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10942 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10943 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10944 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10945 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10946#endif
10947 }
10948 else
10949 {
10950 /* V86 mode checks. */
10951 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10952 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10953 {
10954 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10955 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10956 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10957 }
10958 else
10959 {
10960 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10961 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10962 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10963 }
10964
10965 /* CS */
10966 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10967 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10968 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10969 /* SS */
10970 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10971 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10972 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10973 /* DS */
10974 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10975 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10976 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10977 /* ES */
10978 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10979 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10980 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10981 /* FS */
10982 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10983 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10984 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10985 /* GS */
10986 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10987 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10988 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10989 /* 64-bit capable CPUs. */
10990#if HC_ARCH_BITS == 64
10991 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10992 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10993 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10994 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10995 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10996 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10997 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10998 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10999 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11000 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11001 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11002#endif
11003 }
11004
11005 /*
11006 * TR.
11007 */
11008 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11009 /* 64-bit capable CPUs. */
11010#if HC_ARCH_BITS == 64
11011 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11012#endif
11013 if (fLongModeGuest)
11014 {
11015 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11016 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11017 }
11018 else
11019 {
11020 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11021 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11022 VMX_IGS_TR_ATTR_TYPE_INVALID);
11023 }
11024 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11025 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11026 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11027 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11028 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11029 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11030 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11031 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11032
11033 /*
11034 * GDTR and IDTR.
11035 */
11036#if HC_ARCH_BITS == 64
11037 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11038 AssertRCBreak(rc);
11039 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11040
11041 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11042 AssertRCBreak(rc);
11043 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11044#endif
11045
11046 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11047 AssertRCBreak(rc);
11048 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11049
11050 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11051 AssertRCBreak(rc);
11052 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11053
11054 /*
11055 * Guest Non-Register State.
11056 */
11057 /* Activity State. */
11058 uint32_t u32ActivityState;
11059 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11060 AssertRCBreak(rc);
11061 HMVMX_CHECK_BREAK( !u32ActivityState
11062 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11063 VMX_IGS_ACTIVITY_STATE_INVALID);
11064 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11065 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11066 uint32_t u32IntrState;
11067 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11068 AssertRCBreak(rc);
11069 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11070 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11071 {
11072 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11073 }
11074
11075 /** @todo Activity state and injecting interrupts. Left as a todo since we
11076 * currently don't use activity states but ACTIVE. */
11077
11078 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11079 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11080
11081 /* Guest interruptibility-state. */
11082 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11083 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11084 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11085 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11086 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11087 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11088 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11089 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11090 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11091 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11092 {
11093 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11094 {
11095 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11096 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11097 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11098 }
11099 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11100 {
11101 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11102 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11103 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11104 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11105 }
11106 }
11107 /** @todo Assumes the processor is not in SMM. */
11108 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11109 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11110 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11111 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11112 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11113 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11114 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11115 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11116 {
11117 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11118 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11119 }
11120
11121 /* Pending debug exceptions. */
11122#if HC_ARCH_BITS == 64
11123 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11124 AssertRCBreak(rc);
11125 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11126 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11127 u32Val = u64Val; /* For pending debug exceptions checks below. */
11128#else
11129 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11130 AssertRCBreak(rc);
11131 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11132 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11133#endif
11134
11135 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11136 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11137 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11138 {
11139 if ( (u32Eflags & X86_EFL_TF)
11140 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11141 {
11142 /* Bit 14 is PendingDebug.BS. */
11143 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11144 }
11145 if ( !(u32Eflags & X86_EFL_TF)
11146 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11147 {
11148 /* Bit 14 is PendingDebug.BS. */
11149 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11150 }
11151 }
11152
11153 /* VMCS link pointer. */
11154 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11155 AssertRCBreak(rc);
11156 if (u64Val != UINT64_C(0xffffffffffffffff))
11157 {
11158 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11159 /** @todo Bits beyond the processor's physical-address width MBZ. */
11160 /** @todo 32-bit located in memory referenced by value of this field (as a
11161 * physical address) must contain the processor's VMCS revision ID. */
11162 /** @todo SMM checks. */
11163 }
11164
11165 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11166 * not using Nested Paging? */
11167 if ( pVM->hm.s.fNestedPaging
11168 && !fLongModeGuest
11169 && CPUMIsGuestInPAEModeEx(pCtx))
11170 {
11171 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11172 AssertRCBreak(rc);
11173 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11174
11175 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11176 AssertRCBreak(rc);
11177 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11178
11179 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11180 AssertRCBreak(rc);
11181 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11182
11183 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11184 AssertRCBreak(rc);
11185 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11186 }
11187
11188 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11189 if (uError == VMX_IGS_ERROR)
11190 uError = VMX_IGS_REASON_NOT_FOUND;
11191 } while (0);
11192
11193 pVCpu->hm.s.u32HMError = uError;
11194 return uError;
11195
11196#undef HMVMX_ERROR_BREAK
11197#undef HMVMX_CHECK_BREAK
11198}
11199
11200/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11201/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11202/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11203
11204/** @name VM-exit handlers.
11205 * @{
11206 */
11207
11208/**
11209 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11210 */
11211HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11212{
11213 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11214 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11215 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11216 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11217 return VINF_SUCCESS;
11218 return VINF_EM_RAW_INTERRUPT;
11219}
11220
11221
11222/**
11223 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11224 */
11225HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11226{
11227 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11228 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11229
11230 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11231 AssertRCReturn(rc, rc);
11232
11233 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11234 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11235 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11236 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11237
11238 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11239 {
11240 /*
11241 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11242 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11243 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11244 *
11245 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11246 */
11247 VMXDispatchHostNmi();
11248 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11249 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11250 return VINF_SUCCESS;
11251 }
11252
11253 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11254 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11255 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11256 { /* likely */ }
11257 else
11258 {
11259 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11260 rcStrictRc1 = VINF_SUCCESS;
11261 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11262 return rcStrictRc1;
11263 }
11264
11265 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11266 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11267 switch (uIntType)
11268 {
11269 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11270 Assert(uVector == X86_XCPT_DB);
11271 /* no break */
11272 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11273 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11274 /* no break */
11275 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11276 {
11277 /*
11278 * If there's any exception caused as a result of event injection, go back to
11279 * the interpreter. The page-fault case is complicated and we manually handle
11280 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11281 * handled in hmR0VmxCheckExitDueToEventDelivery.
11282 */
11283 if (!pVCpu->hm.s.Event.fPending)
11284 { /* likely */ }
11285 else if ( uVector != X86_XCPT_PF
11286 && uVector != X86_XCPT_AC)
11287 {
11288 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11289 rc = VERR_EM_INTERPRETER;
11290 break;
11291 }
11292
11293 switch (uVector)
11294 {
11295 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11296 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11297 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11298 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11299 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11300 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11301 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11302
11303 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11304 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11305 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11306 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11307 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11308 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11309 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11310 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11311 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11312 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11313 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11314 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11315 default:
11316 {
11317 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11318 AssertRCReturn(rc, rc);
11319
11320 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11321 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11322 {
11323 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11324 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11325 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11326
11327 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11328 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11329 AssertRCReturn(rc, rc);
11330 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11331 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11332 0 /* GCPtrFaultAddress */);
11333 AssertRCReturn(rc, rc);
11334 }
11335 else
11336 {
11337 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11338 pVCpu->hm.s.u32HMError = uVector;
11339 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11340 }
11341 break;
11342 }
11343 }
11344 break;
11345 }
11346
11347 default:
11348 {
11349 pVCpu->hm.s.u32HMError = uExitIntInfo;
11350 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11351 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11352 break;
11353 }
11354 }
11355 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11356 return rc;
11357}
11358
11359
11360/**
11361 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11362 */
11363HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11364{
11365 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11366
11367 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11368 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11369
11370 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11371 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11372 return VINF_SUCCESS;
11373}
11374
11375
11376/**
11377 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11378 */
11379HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11380{
11381 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11382 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11383 {
11384 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11385 HMVMX_RETURN_UNEXPECTED_EXIT();
11386 }
11387
11388 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11389
11390 /*
11391 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11392 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11393 */
11394 uint32_t uIntrState = 0;
11395 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11396 AssertRCReturn(rc, rc);
11397
11398 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11399 if ( fBlockSti
11400 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11401 {
11402 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11403 }
11404
11405 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11406 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11407
11408 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11409 return VINF_SUCCESS;
11410}
11411
11412
11413/**
11414 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11415 */
11416HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11417{
11418 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11419 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11420 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11421}
11422
11423
11424/**
11425 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11426 */
11427HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11428{
11429 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11430 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11431 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11432}
11433
11434
11435/**
11436 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11437 */
11438HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11439{
11440 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11441 PVM pVM = pVCpu->CTX_SUFF(pVM);
11442 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11443 if (RT_LIKELY(rc == VINF_SUCCESS))
11444 {
11445 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11446 Assert(pVmxTransient->cbInstr == 2);
11447 }
11448 else
11449 {
11450 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11451 rc = VERR_EM_INTERPRETER;
11452 }
11453 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11454 return rc;
11455}
11456
11457
11458/**
11459 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11460 */
11461HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11462{
11463 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11464 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11465 AssertRCReturn(rc, rc);
11466
11467 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11468 return VINF_EM_RAW_EMULATE_INSTR;
11469
11470 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11471 HMVMX_RETURN_UNEXPECTED_EXIT();
11472}
11473
11474
11475/**
11476 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11477 */
11478HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11479{
11480 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11481 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11482 AssertRCReturn(rc, rc);
11483
11484 PVM pVM = pVCpu->CTX_SUFF(pVM);
11485 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11486 if (RT_LIKELY(rc == VINF_SUCCESS))
11487 {
11488 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11489 Assert(pVmxTransient->cbInstr == 2);
11490 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11491 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11492 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11493 }
11494 else
11495 rc = VERR_EM_INTERPRETER;
11496 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11497 return rc;
11498}
11499
11500
11501/**
11502 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11503 */
11504HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11505{
11506 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11507 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11508 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11509 AssertRCReturn(rc, rc);
11510
11511 PVM pVM = pVCpu->CTX_SUFF(pVM);
11512 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11513 if (RT_SUCCESS(rc))
11514 {
11515 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11516 Assert(pVmxTransient->cbInstr == 3);
11517 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11518 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11519 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11520 }
11521 else
11522 {
11523 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11524 rc = VERR_EM_INTERPRETER;
11525 }
11526 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11527 return rc;
11528}
11529
11530
11531/**
11532 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11533 */
11534HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11535{
11536 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11537 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11538 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11539 AssertRCReturn(rc, rc);
11540
11541 PVM pVM = pVCpu->CTX_SUFF(pVM);
11542 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11543 if (RT_LIKELY(rc == VINF_SUCCESS))
11544 {
11545 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11546 Assert(pVmxTransient->cbInstr == 2);
11547 }
11548 else
11549 {
11550 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11551 rc = VERR_EM_INTERPRETER;
11552 }
11553 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11554 return rc;
11555}
11556
11557
11558/**
11559 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11560 */
11561HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11562{
11563 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11564 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11565
11566 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11567 if (pVCpu->hm.s.fHypercallsEnabled)
11568 {
11569#if 0
11570 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11571#else
11572 /* Aggressive state sync. for now. */
11573 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11574 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11575 AssertRCReturn(rc, rc);
11576#endif
11577
11578 /* Perform the hypercall. */
11579 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11580 if (rcStrict == VINF_SUCCESS)
11581 {
11582 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11583 AssertRCReturn(rc, rc);
11584 }
11585 else
11586 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11587 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11588 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11589
11590 /* If the hypercall changes anything other than guest's general-purpose registers,
11591 we would need to reload the guest changed bits here before VM-entry. */
11592 }
11593 else
11594 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11595
11596 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11597 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11598 {
11599 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11600 rcStrict = VINF_SUCCESS;
11601 }
11602
11603 return rcStrict;
11604}
11605
11606
11607/**
11608 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11609 */
11610HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11611{
11612 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11613 PVM pVM = pVCpu->CTX_SUFF(pVM);
11614 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11615
11616 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11617 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11618 AssertRCReturn(rc, rc);
11619
11620 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11621 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11622 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11623 else
11624 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11625 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11626 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11627 return rcStrict;
11628}
11629
11630
11631/**
11632 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11633 */
11634HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11635{
11636 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11637 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11638 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11639 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11640 AssertRCReturn(rc, rc);
11641
11642 PVM pVM = pVCpu->CTX_SUFF(pVM);
11643 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11644 if (RT_LIKELY(rc == VINF_SUCCESS))
11645 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11646 else
11647 {
11648 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11649 rc = VERR_EM_INTERPRETER;
11650 }
11651 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11652 return rc;
11653}
11654
11655
11656/**
11657 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11658 */
11659HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11660{
11661 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11662 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11663 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11664 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11665 AssertRCReturn(rc, rc);
11666
11667 PVM pVM = pVCpu->CTX_SUFF(pVM);
11668 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11669 rc = VBOXSTRICTRC_VAL(rc2);
11670 if (RT_LIKELY( rc == VINF_SUCCESS
11671 || rc == VINF_EM_HALT))
11672 {
11673 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11674 AssertRCReturn(rc3, rc3);
11675
11676 if ( rc == VINF_EM_HALT
11677 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11678 {
11679 rc = VINF_SUCCESS;
11680 }
11681 }
11682 else
11683 {
11684 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11685 rc = VERR_EM_INTERPRETER;
11686 }
11687 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11688 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11689 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11690 return rc;
11691}
11692
11693
11694/**
11695 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11696 */
11697HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11698{
11699 /*
11700 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11701 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11702 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11703 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11704 */
11705 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11706 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11707 HMVMX_RETURN_UNEXPECTED_EXIT();
11708}
11709
11710
11711/**
11712 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11713 */
11714HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11715{
11716 /*
11717 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11718 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11719 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11720 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11721 */
11722 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11723 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11724 HMVMX_RETURN_UNEXPECTED_EXIT();
11725}
11726
11727
11728/**
11729 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11730 */
11731HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11732{
11733 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11734 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11735 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11736 HMVMX_RETURN_UNEXPECTED_EXIT();
11737}
11738
11739
11740/**
11741 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11742 */
11743HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11744{
11745 /*
11746 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11747 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11748 * See Intel spec. 25.3 "Other Causes of VM-exits".
11749 */
11750 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11751 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11752 HMVMX_RETURN_UNEXPECTED_EXIT();
11753}
11754
11755
11756/**
11757 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11758 * VM-exit.
11759 */
11760HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11761{
11762 /*
11763 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11764 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11765 *
11766 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11767 * See Intel spec. "23.8 Restrictions on VMX operation".
11768 */
11769 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11770 return VINF_SUCCESS;
11771}
11772
11773
11774/**
11775 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11776 * VM-exit.
11777 */
11778HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11779{
11780 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11781 return VINF_EM_RESET;
11782}
11783
11784
11785/**
11786 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11787 */
11788HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11789{
11790 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11791 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11792
11793 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11794 AssertRCReturn(rc, rc);
11795
11796 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11797 rc = VINF_SUCCESS;
11798 else
11799 rc = VINF_EM_HALT;
11800
11801 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11802 if (rc != VINF_SUCCESS)
11803 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11804 return rc;
11805}
11806
11807
11808/**
11809 * VM-exit handler for instructions that result in a \#UD exception delivered to
11810 * the guest.
11811 */
11812HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11813{
11814 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11815 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11816 return VINF_SUCCESS;
11817}
11818
11819
11820/**
11821 * VM-exit handler for expiry of the VMX preemption timer.
11822 */
11823HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11824{
11825 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11826
11827 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11828 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11829
11830 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11831 PVM pVM = pVCpu->CTX_SUFF(pVM);
11832 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11833 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11834 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11835}
11836
11837
11838/**
11839 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11840 */
11841HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11842{
11843 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11844
11845 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11846 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11847 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11848 AssertRCReturn(rc, rc);
11849
11850 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11851 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11852
11853 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11854
11855 return rcStrict;
11856}
11857
11858
11859/**
11860 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11861 */
11862HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11863{
11864 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11865
11866 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11867 /** @todo implement EMInterpretInvpcid() */
11868 return VERR_EM_INTERPRETER;
11869}
11870
11871
11872/**
11873 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11874 * Error VM-exit.
11875 */
11876HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11877{
11878 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11879 AssertRCReturn(rc, rc);
11880
11881 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11882 AssertRCReturn(rc, rc);
11883
11884 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11885 NOREF(uInvalidReason);
11886
11887#ifdef VBOX_STRICT
11888 uint32_t uIntrState;
11889 RTHCUINTREG uHCReg;
11890 uint64_t u64Val;
11891 uint32_t u32Val;
11892
11893 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11894 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11895 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11896 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11897 AssertRCReturn(rc, rc);
11898
11899 Log4(("uInvalidReason %u\n", uInvalidReason));
11900 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11901 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11902 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11903 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11904
11905 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11906 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11907 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11908 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11909 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11910 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11911 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11912 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11913 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11914 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11915 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11916 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11917#else
11918 NOREF(pVmxTransient);
11919#endif
11920
11921 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11922 return VERR_VMX_INVALID_GUEST_STATE;
11923}
11924
11925
11926/**
11927 * VM-exit handler for VM-entry failure due to an MSR-load
11928 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11929 */
11930HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11931{
11932 NOREF(pVmxTransient);
11933 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11934 HMVMX_RETURN_UNEXPECTED_EXIT();
11935}
11936
11937
11938/**
11939 * VM-exit handler for VM-entry failure due to a machine-check event
11940 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11941 */
11942HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11943{
11944 NOREF(pVmxTransient);
11945 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11946 HMVMX_RETURN_UNEXPECTED_EXIT();
11947}
11948
11949
11950/**
11951 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11952 * theory.
11953 */
11954HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11955{
11956 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11957 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11958 return VERR_VMX_UNDEFINED_EXIT_CODE;
11959}
11960
11961
11962/**
11963 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11964 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11965 * Conditional VM-exit.
11966 */
11967HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11968{
11969 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11970
11971 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11972 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11973 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11974 return VERR_EM_INTERPRETER;
11975 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11976 HMVMX_RETURN_UNEXPECTED_EXIT();
11977}
11978
11979
11980/**
11981 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11982 */
11983HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11984{
11985 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11986
11987 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11988 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11989 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11990 return VERR_EM_INTERPRETER;
11991 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11992 HMVMX_RETURN_UNEXPECTED_EXIT();
11993}
11994
11995
11996/**
11997 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11998 */
11999HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12000{
12001 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12002
12003 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12004 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12005 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12006 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12007 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12008 {
12009 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12010 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12011 }
12012 AssertRCReturn(rc, rc);
12013 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12014
12015#ifdef VBOX_STRICT
12016 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12017 {
12018 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12019 && pMixedCtx->ecx != MSR_K6_EFER)
12020 {
12021 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12022 pMixedCtx->ecx));
12023 HMVMX_RETURN_UNEXPECTED_EXIT();
12024 }
12025 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12026 {
12027 VMXMSREXITREAD enmRead;
12028 VMXMSREXITWRITE enmWrite;
12029 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12030 AssertRCReturn(rc2, rc2);
12031 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12032 {
12033 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12034 HMVMX_RETURN_UNEXPECTED_EXIT();
12035 }
12036 }
12037 }
12038#endif
12039
12040 PVM pVM = pVCpu->CTX_SUFF(pVM);
12041 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12042 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12043 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12044 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12045 if (RT_SUCCESS(rc))
12046 {
12047 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12048 Assert(pVmxTransient->cbInstr == 2);
12049 }
12050 return rc;
12051}
12052
12053
12054/**
12055 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12056 */
12057HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12058{
12059 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12060 PVM pVM = pVCpu->CTX_SUFF(pVM);
12061 int rc = VINF_SUCCESS;
12062
12063 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12064 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12065 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12066 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12067 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12068 {
12069 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12070 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12071 }
12072 AssertRCReturn(rc, rc);
12073 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12074
12075 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12076 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12077 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12078
12079 if (RT_SUCCESS(rc))
12080 {
12081 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12082
12083 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12084 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12085 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
12086 {
12087 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12088 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12089 EMInterpretWrmsr() changes it. */
12090 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12091 }
12092 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12093 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12094 else if (pMixedCtx->ecx == MSR_K6_EFER)
12095 {
12096 /*
12097 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12098 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12099 * the other bits as well, SCE and NXE. See @bugref{7368}.
12100 */
12101 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12102 }
12103
12104 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12105 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12106 {
12107 switch (pMixedCtx->ecx)
12108 {
12109 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12110 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12111 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12112 case MSR_K8_FS_BASE: /* no break */
12113 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12114 case MSR_K6_EFER: /* already handled above */ break;
12115 default:
12116 {
12117 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12118 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12119 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12120 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12121 break;
12122 }
12123 }
12124 }
12125#ifdef VBOX_STRICT
12126 else
12127 {
12128 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12129 switch (pMixedCtx->ecx)
12130 {
12131 case MSR_IA32_SYSENTER_CS:
12132 case MSR_IA32_SYSENTER_EIP:
12133 case MSR_IA32_SYSENTER_ESP:
12134 case MSR_K8_FS_BASE:
12135 case MSR_K8_GS_BASE:
12136 {
12137 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12138 HMVMX_RETURN_UNEXPECTED_EXIT();
12139 }
12140
12141 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12142 default:
12143 {
12144 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12145 {
12146 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12147 if (pMixedCtx->ecx != MSR_K6_EFER)
12148 {
12149 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12150 pMixedCtx->ecx));
12151 HMVMX_RETURN_UNEXPECTED_EXIT();
12152 }
12153 }
12154
12155 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12156 {
12157 VMXMSREXITREAD enmRead;
12158 VMXMSREXITWRITE enmWrite;
12159 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12160 AssertRCReturn(rc2, rc2);
12161 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12162 {
12163 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12164 HMVMX_RETURN_UNEXPECTED_EXIT();
12165 }
12166 }
12167 break;
12168 }
12169 }
12170 }
12171#endif /* VBOX_STRICT */
12172 }
12173 return rc;
12174}
12175
12176
12177/**
12178 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12179 */
12180HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12181{
12182 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12183
12184 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12185 return VINF_EM_RAW_INTERRUPT;
12186}
12187
12188
12189/**
12190 * VM-exit handler for when the TPR value is lowered below the specified
12191 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12192 */
12193HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12194{
12195 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12196 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12197
12198 /*
12199 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12200 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12201 * resume guest execution.
12202 */
12203 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12204 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12205 return VINF_SUCCESS;
12206}
12207
12208
12209/**
12210 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12211 * VM-exit.
12212 *
12213 * @retval VINF_SUCCESS when guest execution can continue.
12214 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12215 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12216 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12217 * interpreter.
12218 */
12219HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12220{
12221 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12222 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12223 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12224 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12225 AssertRCReturn(rc, rc);
12226
12227 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12228 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12229 PVM pVM = pVCpu->CTX_SUFF(pVM);
12230 VBOXSTRICTRC rcStrict;
12231 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12232 switch (uAccessType)
12233 {
12234 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12235 {
12236 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12237 AssertRCReturn(rc, rc);
12238
12239 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12240 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12241 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12242 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12243 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12244 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12245 {
12246 case 0: /* CR0 */
12247 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12248 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12249 break;
12250 case 2: /* CR2 */
12251 /* Nothing to do here, CR2 it's not part of the VMCS. */
12252 break;
12253 case 3: /* CR3 */
12254 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12255 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12256 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12257 break;
12258 case 4: /* CR4 */
12259 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12260 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12261 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12262 break;
12263 case 8: /* CR8 */
12264 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12265 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12266 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12267 break;
12268 default:
12269 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12270 break;
12271 }
12272
12273 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12274 break;
12275 }
12276
12277 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12278 {
12279 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12280 AssertRCReturn(rc, rc);
12281
12282 Assert( !pVM->hm.s.fNestedPaging
12283 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12284 || pVCpu->hm.s.fUsingDebugLoop
12285 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12286
12287 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12288 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12289 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12290
12291 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12292 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12293 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12294 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12295 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12296 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12297 VBOXSTRICTRC_VAL(rcStrict)));
12298 break;
12299 }
12300
12301 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12302 {
12303 AssertRCReturn(rc, rc);
12304 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12305 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12306 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12307 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12308 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12309 break;
12310 }
12311
12312 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12313 {
12314 AssertRCReturn(rc, rc);
12315 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12316 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12317 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12318 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12319 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12320 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12321 break;
12322 }
12323
12324 default:
12325 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12326 VERR_VMX_UNEXPECTED_EXCEPTION);
12327 }
12328
12329 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12330 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12331 NOREF(pVM);
12332 return rcStrict;
12333}
12334
12335
12336/**
12337 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12338 * VM-exit.
12339 */
12340HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12341{
12342 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12343 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12344
12345 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12346 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12347 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12348 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12349 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12350 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12351 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12352 AssertRCReturn(rc2, rc2);
12353
12354 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12355 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12356 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12357 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12358 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12359 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12360 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12361 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12362 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12363
12364 /* I/O operation lookup arrays. */
12365 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12366 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12367
12368 VBOXSTRICTRC rcStrict;
12369 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12370 uint32_t const cbInstr = pVmxTransient->cbInstr;
12371 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12372 PVM pVM = pVCpu->CTX_SUFF(pVM);
12373 if (fIOString)
12374 {
12375#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12376 See @bugref{5752#c158}. Should work now. */
12377 /*
12378 * INS/OUTS - I/O String instruction.
12379 *
12380 * Use instruction-information if available, otherwise fall back on
12381 * interpreting the instruction.
12382 */
12383 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12384 fIOWrite ? 'w' : 'r'));
12385 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12386 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12387 {
12388 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12389 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12390 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12391 AssertRCReturn(rc2, rc2);
12392 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12393 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12394 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12395 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12396 if (fIOWrite)
12397 {
12398 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12399 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12400 }
12401 else
12402 {
12403 /*
12404 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12405 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12406 * See Intel Instruction spec. for "INS".
12407 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12408 */
12409 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12410 }
12411 }
12412 else
12413 {
12414 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12415 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12416 AssertRCReturn(rc2, rc2);
12417 rcStrict = IEMExecOne(pVCpu);
12418 }
12419 /** @todo IEM needs to be setting these flags somehow. */
12420 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12421 fUpdateRipAlready = true;
12422#else
12423 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12424 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12425 if (RT_SUCCESS(rcStrict))
12426 {
12427 if (fIOWrite)
12428 {
12429 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12430 (DISCPUMODE)pDis->uAddrMode, cbValue);
12431 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12432 }
12433 else
12434 {
12435 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12436 (DISCPUMODE)pDis->uAddrMode, cbValue);
12437 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12438 }
12439 }
12440 else
12441 {
12442 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12443 pMixedCtx->rip));
12444 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12445 }
12446#endif
12447 }
12448 else
12449 {
12450 /*
12451 * IN/OUT - I/O instruction.
12452 */
12453 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12454 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12455 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12456 if (fIOWrite)
12457 {
12458 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12459 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12460 }
12461 else
12462 {
12463 uint32_t u32Result = 0;
12464 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12465 if (IOM_SUCCESS(rcStrict))
12466 {
12467 /* Save result of I/O IN instr. in AL/AX/EAX. */
12468 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12469 }
12470 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12471 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12472 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12473 }
12474 }
12475
12476 if (IOM_SUCCESS(rcStrict))
12477 {
12478 if (!fUpdateRipAlready)
12479 {
12480 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12481 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12482 }
12483
12484 /*
12485 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12486 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12487 */
12488 if (fIOString)
12489 {
12490 /** @todo Single-step for INS/OUTS with REP prefix? */
12491 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12492 }
12493 else if ( !fDbgStepping
12494 && fGstStepping)
12495 {
12496 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12497 }
12498
12499 /*
12500 * If any I/O breakpoints are armed, we need to check if one triggered
12501 * and take appropriate action.
12502 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12503 */
12504 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12505 AssertRCReturn(rc2, rc2);
12506
12507 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12508 * execution engines about whether hyper BPs and such are pending. */
12509 uint32_t const uDr7 = pMixedCtx->dr[7];
12510 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12511 && X86_DR7_ANY_RW_IO(uDr7)
12512 && (pMixedCtx->cr4 & X86_CR4_DE))
12513 || DBGFBpIsHwIoArmed(pVM)))
12514 {
12515 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12516
12517 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12518 VMMRZCallRing3Disable(pVCpu);
12519 HM_DISABLE_PREEMPT();
12520
12521 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12522
12523 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12524 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12525 {
12526 /* Raise #DB. */
12527 if (fIsGuestDbgActive)
12528 ASMSetDR6(pMixedCtx->dr[6]);
12529 if (pMixedCtx->dr[7] != uDr7)
12530 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12531
12532 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12533 }
12534 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12535 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12536 else if ( rcStrict2 != VINF_SUCCESS
12537 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12538 rcStrict = rcStrict2;
12539 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12540
12541 HM_RESTORE_PREEMPT();
12542 VMMRZCallRing3Enable(pVCpu);
12543 }
12544 }
12545
12546#ifdef VBOX_STRICT
12547 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12548 Assert(!fIOWrite);
12549 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12550 Assert(fIOWrite);
12551 else
12552 {
12553#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12554 * statuses, that the VMM device and some others may return. See
12555 * IOM_SUCCESS() for guidance. */
12556 AssertMsg( RT_FAILURE(rcStrict)
12557 || rcStrict == VINF_SUCCESS
12558 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12559 || rcStrict == VINF_EM_DBG_BREAKPOINT
12560 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12561 || rcStrict == VINF_EM_RAW_TO_R3
12562 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12563#endif
12564 }
12565#endif
12566
12567 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12568 return rcStrict;
12569}
12570
12571
12572/**
12573 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12574 * VM-exit.
12575 */
12576HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12577{
12578 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12579
12580 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12581 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12582 AssertRCReturn(rc, rc);
12583 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12584 {
12585 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12586 AssertRCReturn(rc, rc);
12587 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12588 {
12589 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12590
12591 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12592 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12593
12594 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12595 Assert(!pVCpu->hm.s.Event.fPending);
12596 pVCpu->hm.s.Event.fPending = true;
12597 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12598 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12599 AssertRCReturn(rc, rc);
12600 if (fErrorCodeValid)
12601 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12602 else
12603 pVCpu->hm.s.Event.u32ErrCode = 0;
12604 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12605 && uVector == X86_XCPT_PF)
12606 {
12607 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12608 }
12609
12610 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12611 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12612 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12613 }
12614 }
12615
12616 /* Fall back to the interpreter to emulate the task-switch. */
12617 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12618 return VERR_EM_INTERPRETER;
12619}
12620
12621
12622/**
12623 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12624 */
12625HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12626{
12627 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12628 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12629 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12630 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12631 AssertRCReturn(rc, rc);
12632 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12633 return VINF_EM_DBG_STEPPED;
12634}
12635
12636
12637/**
12638 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12639 */
12640HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12641{
12642 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12643
12644 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12645
12646 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12647 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12648 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12649 {
12650 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12651 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12652 {
12653 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12654 return VERR_EM_INTERPRETER;
12655 }
12656 }
12657 else
12658 {
12659 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12660 rcStrict1 = VINF_SUCCESS;
12661 return rcStrict1;
12662 }
12663
12664#if 0
12665 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12666 * just sync the whole thing. */
12667 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12668#else
12669 /* Aggressive state sync. for now. */
12670 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12671 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12672 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12673#endif
12674 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12675 AssertRCReturn(rc, rc);
12676
12677 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12678 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12679 VBOXSTRICTRC rcStrict2;
12680 switch (uAccessType)
12681 {
12682 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12683 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12684 {
12685 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12686 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12687 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12688
12689 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12690 GCPhys &= PAGE_BASE_GC_MASK;
12691 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12692 PVM pVM = pVCpu->CTX_SUFF(pVM);
12693 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12694 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12695
12696 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12697 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12698 CPUMCTX2CORE(pMixedCtx), GCPhys);
12699 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12700 if ( rcStrict2 == VINF_SUCCESS
12701 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12702 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12703 {
12704 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12705 | HM_CHANGED_GUEST_RSP
12706 | HM_CHANGED_GUEST_RFLAGS
12707 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12708 rcStrict2 = VINF_SUCCESS;
12709 }
12710 break;
12711 }
12712
12713 default:
12714 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12715 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12716 break;
12717 }
12718
12719 if (rcStrict2 != VINF_SUCCESS)
12720 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12721 return rcStrict2;
12722}
12723
12724
12725/**
12726 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12727 * VM-exit.
12728 */
12729HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12730{
12731 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12732
12733 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12734 if (pVmxTransient->fWasGuestDebugStateActive)
12735 {
12736 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12737 HMVMX_RETURN_UNEXPECTED_EXIT();
12738 }
12739
12740 if ( !pVCpu->hm.s.fSingleInstruction
12741 && !pVmxTransient->fWasHyperDebugStateActive)
12742 {
12743 Assert(!DBGFIsStepping(pVCpu));
12744 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12745
12746 /* Don't intercept MOV DRx any more. */
12747 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12748 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12749 AssertRCReturn(rc, rc);
12750
12751 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12752 VMMRZCallRing3Disable(pVCpu);
12753 HM_DISABLE_PREEMPT();
12754
12755 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12756 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12757 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12758
12759 HM_RESTORE_PREEMPT();
12760 VMMRZCallRing3Enable(pVCpu);
12761
12762#ifdef VBOX_WITH_STATISTICS
12763 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12764 AssertRCReturn(rc, rc);
12765 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12766 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12767 else
12768 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12769#endif
12770 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12771 return VINF_SUCCESS;
12772 }
12773
12774 /*
12775 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12776 * Update the segment registers and DR7 from the CPU.
12777 */
12778 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12779 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12780 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12781 AssertRCReturn(rc, rc);
12782 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12783
12784 PVM pVM = pVCpu->CTX_SUFF(pVM);
12785 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12786 {
12787 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12788 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12789 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12790 if (RT_SUCCESS(rc))
12791 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12792 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12793 }
12794 else
12795 {
12796 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12797 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12798 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12800 }
12801
12802 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12803 if (RT_SUCCESS(rc))
12804 {
12805 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12806 AssertRCReturn(rc2, rc2);
12807 return VINF_SUCCESS;
12808 }
12809 return rc;
12810}
12811
12812
12813/**
12814 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12815 * Conditional VM-exit.
12816 */
12817HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12818{
12819 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12820 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12821
12822 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12823 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12824 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12825 {
12826 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12827 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12828 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12829 {
12830 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12831 return VERR_EM_INTERPRETER;
12832 }
12833 }
12834 else
12835 {
12836 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12837 rcStrict1 = VINF_SUCCESS;
12838 return rcStrict1;
12839 }
12840
12841 RTGCPHYS GCPhys = 0;
12842 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12843
12844#if 0
12845 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12846#else
12847 /* Aggressive state sync. for now. */
12848 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12849 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12850 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12851#endif
12852 AssertRCReturn(rc, rc);
12853
12854 /*
12855 * If we succeed, resume guest execution.
12856 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12857 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12858 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12859 * weird case. See @bugref{6043}.
12860 */
12861 PVM pVM = pVCpu->CTX_SUFF(pVM);
12862 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12863 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12864 if ( rcStrict2 == VINF_SUCCESS
12865 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12866 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12867 {
12868 /* Successfully handled MMIO operation. */
12869 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12870 | HM_CHANGED_GUEST_RSP
12871 | HM_CHANGED_GUEST_RFLAGS
12872 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12873 return VINF_SUCCESS;
12874 }
12875 return rcStrict2;
12876}
12877
12878
12879/**
12880 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12881 * VM-exit.
12882 */
12883HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12884{
12885 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12886 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12887
12888 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12889 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12890 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12891 {
12892 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12893 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12894 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12895 }
12896 else
12897 {
12898 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12899 rcStrict1 = VINF_SUCCESS;
12900 return rcStrict1;
12901 }
12902
12903 RTGCPHYS GCPhys = 0;
12904 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12905 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12906#if 0
12907 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12908#else
12909 /* Aggressive state sync. for now. */
12910 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12911 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12912 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12913#endif
12914 AssertRCReturn(rc, rc);
12915
12916 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12917 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12918
12919 RTGCUINT uErrorCode = 0;
12920 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12921 uErrorCode |= X86_TRAP_PF_ID;
12922 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12923 uErrorCode |= X86_TRAP_PF_RW;
12924 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12925 uErrorCode |= X86_TRAP_PF_P;
12926
12927 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12928
12929 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12930 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12931
12932 /* Handle the pagefault trap for the nested shadow table. */
12933 PVM pVM = pVCpu->CTX_SUFF(pVM);
12934 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12935 TRPMResetTrap(pVCpu);
12936
12937 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12938 if ( rcStrict2 == VINF_SUCCESS
12939 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12940 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12941 {
12942 /* Successfully synced our nested page tables. */
12943 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12944 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12945 | HM_CHANGED_GUEST_RSP
12946 | HM_CHANGED_GUEST_RFLAGS);
12947 return VINF_SUCCESS;
12948 }
12949
12950 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12951 return rcStrict2;
12952}
12953
12954/** @} */
12955
12956/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12957/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12958/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12959
12960/** @name VM-exit exception handlers.
12961 * @{
12962 */
12963
12964/**
12965 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12966 */
12967static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12968{
12969 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12971
12972 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12973 AssertRCReturn(rc, rc);
12974
12975 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12976 {
12977 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12978 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12979
12980 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12981 * provides VM-exit instruction length. If this causes problem later,
12982 * disassemble the instruction like it's done on AMD-V. */
12983 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12984 AssertRCReturn(rc2, rc2);
12985 return rc;
12986 }
12987
12988 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12989 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12990 return rc;
12991}
12992
12993
12994/**
12995 * VM-exit exception handler for \#BP (Breakpoint exception).
12996 */
12997static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12998{
12999 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13001
13002 /** @todo Try optimize this by not saving the entire guest state unless
13003 * really needed. */
13004 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13005 AssertRCReturn(rc, rc);
13006
13007 PVM pVM = pVCpu->CTX_SUFF(pVM);
13008 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13009 if (rc == VINF_EM_RAW_GUEST_TRAP)
13010 {
13011 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13012 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13013 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13014 AssertRCReturn(rc, rc);
13015
13016 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13017 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13018 }
13019
13020 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13021 return rc;
13022}
13023
13024
13025/**
13026 * VM-exit exception handler for \#AC (alignment check exception).
13027 */
13028static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13029{
13030 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13031
13032 /*
13033 * Re-inject it. We'll detect any nesting before getting here.
13034 */
13035 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13036 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13037 AssertRCReturn(rc, rc);
13038 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13039
13040 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13041 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13042 return VINF_SUCCESS;
13043}
13044
13045
13046/**
13047 * VM-exit exception handler for \#DB (Debug exception).
13048 */
13049static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13050{
13051 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13052 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13053 Log6(("XcptDB\n"));
13054
13055 /*
13056 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13057 * for processing.
13058 */
13059 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13060 AssertRCReturn(rc, rc);
13061
13062 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13063 uint64_t uDR6 = X86_DR6_INIT_VAL;
13064 uDR6 |= ( pVmxTransient->uExitQualification
13065 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13066
13067 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13068 if (rc == VINF_EM_RAW_GUEST_TRAP)
13069 {
13070 /*
13071 * The exception was for the guest. Update DR6, DR7.GD and
13072 * IA32_DEBUGCTL.LBR before forwarding it.
13073 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13074 */
13075 VMMRZCallRing3Disable(pVCpu);
13076 HM_DISABLE_PREEMPT();
13077
13078 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13079 pMixedCtx->dr[6] |= uDR6;
13080 if (CPUMIsGuestDebugStateActive(pVCpu))
13081 ASMSetDR6(pMixedCtx->dr[6]);
13082
13083 HM_RESTORE_PREEMPT();
13084 VMMRZCallRing3Enable(pVCpu);
13085
13086 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13087 AssertRCReturn(rc, rc);
13088
13089 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13090 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13091
13092 /* Paranoia. */
13093 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13094 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13095
13096 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13097 AssertRCReturn(rc, rc);
13098
13099 /*
13100 * Raise #DB in the guest.
13101 *
13102 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13103 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13104 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13105 *
13106 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13107 */
13108 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13109 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13110 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13111 AssertRCReturn(rc, rc);
13112 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13113 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13114 return VINF_SUCCESS;
13115 }
13116
13117 /*
13118 * Not a guest trap, must be a hypervisor related debug event then.
13119 * Update DR6 in case someone is interested in it.
13120 */
13121 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13122 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13123 CPUMSetHyperDR6(pVCpu, uDR6);
13124
13125 return rc;
13126}
13127
13128
13129/**
13130 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13131 * point exception).
13132 */
13133static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13134{
13135 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13136
13137 /* We require CR0 and EFER. EFER is always up-to-date. */
13138 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13139 AssertRCReturn(rc, rc);
13140
13141 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13142 VMMRZCallRing3Disable(pVCpu);
13143 HM_DISABLE_PREEMPT();
13144
13145 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
13146 if (pVmxTransient->fWasGuestFPUStateActive)
13147 {
13148 rc = VINF_EM_RAW_GUEST_TRAP;
13149 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13150 }
13151 else
13152 {
13153#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13154 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13155#endif
13156 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13157 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13158 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13159 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13160 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13161 }
13162
13163 HM_RESTORE_PREEMPT();
13164 VMMRZCallRing3Enable(pVCpu);
13165
13166 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13167 {
13168 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13169 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13171 pVCpu->hm.s.fPreloadGuestFpu = true;
13172 }
13173 else
13174 {
13175 /* Forward #NM to the guest. */
13176 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13177 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13178 AssertRCReturn(rc, rc);
13179 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13180 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13181 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13182 }
13183
13184 return VINF_SUCCESS;
13185}
13186
13187
13188/**
13189 * VM-exit exception handler for \#GP (General-protection exception).
13190 *
13191 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13192 */
13193static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13194{
13195 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13196 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13197
13198 int rc;
13199 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13200 { /* likely */ }
13201 else
13202 {
13203#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13204 Assert(pVCpu->hm.s.fUsingDebugLoop);
13205#endif
13206 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13207 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13208 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13209 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13210 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13211 AssertRCReturn(rc, rc);
13212 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13213 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13214 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13215 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13216 return rc;
13217 }
13218
13219 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13220 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13221
13222 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13223 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13224 AssertRCReturn(rc, rc);
13225
13226 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13227 uint32_t cbOp = 0;
13228 PVM pVM = pVCpu->CTX_SUFF(pVM);
13229 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13230 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13231 if (RT_SUCCESS(rc))
13232 {
13233 rc = VINF_SUCCESS;
13234 Assert(cbOp == pDis->cbInstr);
13235 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13236 switch (pDis->pCurInstr->uOpcode)
13237 {
13238 case OP_CLI:
13239 {
13240 pMixedCtx->eflags.Bits.u1IF = 0;
13241 pMixedCtx->eflags.Bits.u1RF = 0;
13242 pMixedCtx->rip += pDis->cbInstr;
13243 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13244 if ( !fDbgStepping
13245 && pMixedCtx->eflags.Bits.u1TF)
13246 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13247 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13248 break;
13249 }
13250
13251 case OP_STI:
13252 {
13253 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13254 pMixedCtx->eflags.Bits.u1IF = 1;
13255 pMixedCtx->eflags.Bits.u1RF = 0;
13256 pMixedCtx->rip += pDis->cbInstr;
13257 if (!fOldIF)
13258 {
13259 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13260 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13261 }
13262 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13263 if ( !fDbgStepping
13264 && pMixedCtx->eflags.Bits.u1TF)
13265 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13266 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13267 break;
13268 }
13269
13270 case OP_HLT:
13271 {
13272 rc = VINF_EM_HALT;
13273 pMixedCtx->rip += pDis->cbInstr;
13274 pMixedCtx->eflags.Bits.u1RF = 0;
13275 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13276 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13277 break;
13278 }
13279
13280 case OP_POPF:
13281 {
13282 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13283 uint32_t cbParm;
13284 uint32_t uMask;
13285 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13286 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13287 {
13288 cbParm = 4;
13289 uMask = 0xffffffff;
13290 }
13291 else
13292 {
13293 cbParm = 2;
13294 uMask = 0xffff;
13295 }
13296
13297 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13298 RTGCPTR GCPtrStack = 0;
13299 X86EFLAGS Eflags;
13300 Eflags.u32 = 0;
13301 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13302 &GCPtrStack);
13303 if (RT_SUCCESS(rc))
13304 {
13305 Assert(sizeof(Eflags.u32) >= cbParm);
13306 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13307 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13308 }
13309 if (RT_FAILURE(rc))
13310 {
13311 rc = VERR_EM_INTERPRETER;
13312 break;
13313 }
13314 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13315 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13316 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13317 pMixedCtx->esp += cbParm;
13318 pMixedCtx->esp &= uMask;
13319 pMixedCtx->rip += pDis->cbInstr;
13320 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13321 | HM_CHANGED_GUEST_RSP
13322 | HM_CHANGED_GUEST_RFLAGS);
13323 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13324 POPF restores EFLAGS.TF. */
13325 if ( !fDbgStepping
13326 && fGstStepping)
13327 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13328 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13329 break;
13330 }
13331
13332 case OP_PUSHF:
13333 {
13334 uint32_t cbParm;
13335 uint32_t uMask;
13336 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13337 {
13338 cbParm = 4;
13339 uMask = 0xffffffff;
13340 }
13341 else
13342 {
13343 cbParm = 2;
13344 uMask = 0xffff;
13345 }
13346
13347 /* Get the stack pointer & push the contents of eflags onto the stack. */
13348 RTGCPTR GCPtrStack = 0;
13349 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13350 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13351 if (RT_FAILURE(rc))
13352 {
13353 rc = VERR_EM_INTERPRETER;
13354 break;
13355 }
13356 X86EFLAGS Eflags = pMixedCtx->eflags;
13357 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13358 Eflags.Bits.u1RF = 0;
13359 Eflags.Bits.u1VM = 0;
13360
13361 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13362 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13363 {
13364 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13365 rc = VERR_EM_INTERPRETER;
13366 break;
13367 }
13368 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13369 pMixedCtx->esp -= cbParm;
13370 pMixedCtx->esp &= uMask;
13371 pMixedCtx->rip += pDis->cbInstr;
13372 pMixedCtx->eflags.Bits.u1RF = 0;
13373 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13374 | HM_CHANGED_GUEST_RSP
13375 | HM_CHANGED_GUEST_RFLAGS);
13376 if ( !fDbgStepping
13377 && pMixedCtx->eflags.Bits.u1TF)
13378 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13379 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13380 break;
13381 }
13382
13383 case OP_IRET:
13384 {
13385 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13386 * instruction reference. */
13387 RTGCPTR GCPtrStack = 0;
13388 uint32_t uMask = 0xffff;
13389 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13390 uint16_t aIretFrame[3];
13391 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13392 {
13393 rc = VERR_EM_INTERPRETER;
13394 break;
13395 }
13396 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13397 &GCPtrStack);
13398 if (RT_SUCCESS(rc))
13399 {
13400 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13401 PGMACCESSORIGIN_HM));
13402 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13403 }
13404 if (RT_FAILURE(rc))
13405 {
13406 rc = VERR_EM_INTERPRETER;
13407 break;
13408 }
13409 pMixedCtx->eip = 0;
13410 pMixedCtx->ip = aIretFrame[0];
13411 pMixedCtx->cs.Sel = aIretFrame[1];
13412 pMixedCtx->cs.ValidSel = aIretFrame[1];
13413 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13414 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13415 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13416 pMixedCtx->sp += sizeof(aIretFrame);
13417 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13418 | HM_CHANGED_GUEST_SEGMENT_REGS
13419 | HM_CHANGED_GUEST_RSP
13420 | HM_CHANGED_GUEST_RFLAGS);
13421 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13422 if ( !fDbgStepping
13423 && fGstStepping)
13424 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13425 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13426 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13427 break;
13428 }
13429
13430 case OP_INT:
13431 {
13432 uint16_t uVector = pDis->Param1.uValue & 0xff;
13433 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13434 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13435 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13436 break;
13437 }
13438
13439 case OP_INTO:
13440 {
13441 if (pMixedCtx->eflags.Bits.u1OF)
13442 {
13443 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13444 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13445 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13446 }
13447 else
13448 {
13449 pMixedCtx->eflags.Bits.u1RF = 0;
13450 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13451 }
13452 break;
13453 }
13454
13455 default:
13456 {
13457 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13458 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13459 EMCODETYPE_SUPERVISOR);
13460 rc = VBOXSTRICTRC_VAL(rc2);
13461 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13462 /** @todo We have to set pending-debug exceptions here when the guest is
13463 * single-stepping depending on the instruction that was interpreted. */
13464 Log4(("#GP rc=%Rrc\n", rc));
13465 break;
13466 }
13467 }
13468 }
13469 else
13470 rc = VERR_EM_INTERPRETER;
13471
13472 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13473 ("#GP Unexpected rc=%Rrc\n", rc));
13474 return rc;
13475}
13476
13477
13478/**
13479 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13480 * the exception reported in the VMX transient structure back into the VM.
13481 *
13482 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13483 * up-to-date.
13484 */
13485static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13486{
13487 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13488#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13489 Assert(pVCpu->hm.s.fUsingDebugLoop);
13490#endif
13491
13492 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13493 hmR0VmxCheckExitDueToEventDelivery(). */
13494 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13495 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13496 AssertRCReturn(rc, rc);
13497 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13498
13499#ifdef DEBUG_ramshankar
13500 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13501 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13502 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13503#endif
13504
13505 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13506 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13507 return VINF_SUCCESS;
13508}
13509
13510
13511/**
13512 * VM-exit exception handler for \#PF (Page-fault exception).
13513 */
13514static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13515{
13516 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13517 PVM pVM = pVCpu->CTX_SUFF(pVM);
13518 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13519 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13520 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13521 AssertRCReturn(rc, rc);
13522
13523 if (!pVM->hm.s.fNestedPaging)
13524 { /* likely */ }
13525 else
13526 {
13527#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13528 Assert(pVCpu->hm.s.fUsingDebugLoop);
13529#endif
13530 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13531 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13532 {
13533 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13534 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13535 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13536 }
13537 else
13538 {
13539 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13540 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13541 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13542 }
13543 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13544 return rc;
13545 }
13546
13547 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13548 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13549 if (pVmxTransient->fVectoringPF)
13550 {
13551 Assert(pVCpu->hm.s.Event.fPending);
13552 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13553 }
13554
13555 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13556 AssertRCReturn(rc, rc);
13557
13558 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13559 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13560
13561 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13562 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13563 (RTGCPTR)pVmxTransient->uExitQualification);
13564
13565 Log4(("#PF: rc=%Rrc\n", rc));
13566 if (rc == VINF_SUCCESS)
13567 {
13568#if 0
13569 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13570 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13571 * memory? We don't update the whole state here... */
13572 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13573 | HM_CHANGED_GUEST_RSP
13574 | HM_CHANGED_GUEST_RFLAGS
13575 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13576#else
13577 /*
13578 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13579 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13580 */
13581 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13582 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13583#endif
13584 TRPMResetTrap(pVCpu);
13585 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13586 return rc;
13587 }
13588
13589 if (rc == VINF_EM_RAW_GUEST_TRAP)
13590 {
13591 if (!pVmxTransient->fVectoringDoublePF)
13592 {
13593 /* It's a guest page fault and needs to be reflected to the guest. */
13594 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13595 TRPMResetTrap(pVCpu);
13596 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13597 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13598 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13599 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13600 }
13601 else
13602 {
13603 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13604 TRPMResetTrap(pVCpu);
13605 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13606 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13607 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13608 }
13609
13610 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13611 return VINF_SUCCESS;
13612 }
13613
13614 TRPMResetTrap(pVCpu);
13615 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13616 return rc;
13617}
13618
13619/** @} */
13620
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette