VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 65381

Last change on this file since 65381 was 65355, checked in by vboxsync, 8 years ago

VMM/HMVMXR0: Save guest rflags before handling any hypercalls.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 585.4 KB
Line 
1/* $Id: HMVMXR0.cpp 65355 2017-01-18 03:48:43Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#ifdef DEBUG_ramshankar
44# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49# define HMVMX_ALWAYS_TRAP_PF
50# define HMVMX_ALWAYS_SWAP_FPU_STATE
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name Updated-guest-state flags.
69 * @{ */
70#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91 | HMVMX_UPDATED_GUEST_RSP \
92 | HMVMX_UPDATED_GUEST_RFLAGS \
93 | HMVMX_UPDATED_GUEST_CR0 \
94 | HMVMX_UPDATED_GUEST_CR3 \
95 | HMVMX_UPDATED_GUEST_CR4 \
96 | HMVMX_UPDATED_GUEST_GDTR \
97 | HMVMX_UPDATED_GUEST_IDTR \
98 | HMVMX_UPDATED_GUEST_LDTR \
99 | HMVMX_UPDATED_GUEST_TR \
100 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101 | HMVMX_UPDATED_GUEST_DEBUG \
102 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
107 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108 | HMVMX_UPDATED_GUEST_INTR_STATE \
109 | HMVMX_UPDATED_GUEST_APIC_STATE)
110/** @} */
111
112/** @name
113 * Flags to skip redundant reads of some common VMCS fields that are not part of
114 * the guest-CPU state but are in the transient structure.
115 */
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123/** @} */
124
125/** @name
126 * States of the VMCS.
127 *
128 * This does not reflect all possible VMCS states but currently only those
129 * needed for maintaining the VMCS consistently even when thread-context hooks
130 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
131 */
132#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135/** @} */
136
137/**
138 * Exception bitmap mask for real-mode guests (real-on-v86).
139 *
140 * We need to intercept all exceptions manually except:
141 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
142 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143 * due to bugs in Intel CPUs.
144 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
145 * support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/** Helper macro for VM-exit handlers called unexpectedly. */
188#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189 do { \
190 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191 return VERR_VMX_UNEXPECTED_EXIT; \
192 } while (0)
193
194
195/*********************************************************************************************************************************
196* Structures and Typedefs *
197*********************************************************************************************************************************/
198/**
199 * VMX transient state.
200 *
201 * A state structure for holding miscellaneous information across
202 * VMX non-root operation and restored after the transition.
203 */
204typedef struct VMXTRANSIENT
205{
206 /** The host's rflags/eflags. */
207 RTCCUINTREG fEFlags;
208#if HC_ARCH_BITS == 32
209 uint32_t u32Alignment0;
210#endif
211 /** The guest's TPR value used for TPR shadowing. */
212 uint8_t u8GuestTpr;
213 /** Alignment. */
214 uint8_t abAlignment0[7];
215
216 /** The basic VM-exit reason. */
217 uint16_t uExitReason;
218 /** Alignment. */
219 uint16_t u16Alignment0;
220 /** The VM-exit interruption error code. */
221 uint32_t uExitIntErrorCode;
222 /** The VM-exit exit code qualification. */
223 uint64_t uExitQualification;
224
225 /** The VM-exit interruption-information field. */
226 uint32_t uExitIntInfo;
227 /** The VM-exit instruction-length field. */
228 uint32_t cbInstr;
229 /** The VM-exit instruction-information field. */
230 union
231 {
232 /** Plain unsigned int representation. */
233 uint32_t u;
234 /** INS and OUTS information. */
235 struct
236 {
237 uint32_t u7Reserved0 : 7;
238 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239 uint32_t u3AddrSize : 3;
240 uint32_t u5Reserved1 : 5;
241 /** The segment register (X86_SREG_XXX). */
242 uint32_t iSegReg : 3;
243 uint32_t uReserved2 : 14;
244 } StrIo;
245 } ExitInstrInfo;
246 /** Whether the VM-entry failed or not. */
247 bool fVMEntryFailed;
248 /** Alignment. */
249 uint8_t abAlignment1[3];
250
251 /** The VM-entry interruption-information field. */
252 uint32_t uEntryIntInfo;
253 /** The VM-entry exception error code field. */
254 uint32_t uEntryXcptErrorCode;
255 /** The VM-entry instruction length field. */
256 uint32_t cbEntryInstr;
257
258 /** IDT-vectoring information field. */
259 uint32_t uIdtVectoringInfo;
260 /** IDT-vectoring error code. */
261 uint32_t uIdtVectoringErrorCode;
262
263 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
264 uint32_t fVmcsFieldsRead;
265
266 /** Whether the guest FPU was active at the time of VM-exit. */
267 bool fWasGuestFPUStateActive;
268 /** Whether the guest debug state was active at the time of VM-exit. */
269 bool fWasGuestDebugStateActive;
270 /** Whether the hyper debug state was active at the time of VM-exit. */
271 bool fWasHyperDebugStateActive;
272 /** Whether TSC-offsetting should be setup before VM-entry. */
273 bool fUpdateTscOffsettingAndPreemptTimer;
274 /** Whether the VM-exit was caused by a page-fault during delivery of a
275 * contributory exception or a page-fault. */
276 bool fVectoringDoublePF;
277 /** Whether the VM-exit was caused by a page-fault during delivery of an
278 * external interrupt or NMI. */
279 bool fVectoringPF;
280} VMXTRANSIENT;
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286/** Pointer to VMX transient state. */
287typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290/**
291 * MSR-bitmap read permissions.
292 */
293typedef enum VMXMSREXITREAD
294{
295 /** Reading this MSR causes a VM-exit. */
296 VMXMSREXIT_INTERCEPT_READ = 0xb,
297 /** Reading this MSR does not cause a VM-exit. */
298 VMXMSREXIT_PASSTHRU_READ
299} VMXMSREXITREAD;
300/** Pointer to MSR-bitmap read permissions. */
301typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303/**
304 * MSR-bitmap write permissions.
305 */
306typedef enum VMXMSREXITWRITE
307{
308 /** Writing to this MSR causes a VM-exit. */
309 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310 /** Writing to this MSR does not cause a VM-exit. */
311 VMXMSREXIT_PASSTHRU_WRITE
312} VMXMSREXITWRITE;
313/** Pointer to MSR-bitmap write permissions. */
314typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317/**
318 * VMX VM-exit handler.
319 *
320 * @returns Strict VBox status code (i.e. informational status codes too).
321 * @param pVCpu The cross context virtual CPU structure.
322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
323 * out-of-sync. Make sure to update the required
324 * fields before using them.
325 * @param pVmxTransient Pointer to the VMX-transient structure.
326 */
327#ifndef HMVMX_USE_FUNCTION_TABLE
328typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329#else
330typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331/** Pointer to VM-exit handler. */
332typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333#endif
334
335/**
336 * VMX VM-exit handler, non-strict status code.
337 *
338 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
339 *
340 * @returns VBox status code, no informational status code returned.
341 * @param pVCpu The cross context virtual CPU structure.
342 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
343 * out-of-sync. Make sure to update the required
344 * fields before using them.
345 * @param pVmxTransient Pointer to the VMX-transient structure.
346 *
347 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
348 * use of that status code will be replaced with VINF_EM_SOMETHING
349 * later when switching over to IEM.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353#else
354typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
355#endif
356
357
358/*********************************************************************************************************************************
359* Internal Functions *
360*********************************************************************************************************************************/
361static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
362static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
363static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
364static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
365 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
366 bool fStepping, uint32_t *puIntState);
367#if HC_ARCH_BITS == 32
368static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
369#endif
370#ifndef HMVMX_USE_FUNCTION_TABLE
371DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
372# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
373# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
374#else
375# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
376# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
377#endif
378
379
380/** @name VM-exit handlers.
381 * @{
382 */
383static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395static FNVMXEXITHANDLER hmR0VmxExitHlt;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411static FNVMXEXITHANDLER hmR0VmxExitMwait;
412static FNVMXEXITHANDLER hmR0VmxExitMtf;
413static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414static FNVMXEXITHANDLER hmR0VmxExitPause;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428/** @} */
429
430static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
439
440
441/*********************************************************************************************************************************
442* Global Variables *
443*********************************************************************************************************************************/
444#ifdef HMVMX_USE_FUNCTION_TABLE
445
446/**
447 * VMX_EXIT dispatch table.
448 */
449static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
450{
451 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
452 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
453 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
454 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
455 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
456 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
457 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
458 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
459 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
460 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
461 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
462 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
463 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
464 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
465 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
466 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
467 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
468 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
469 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
470 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
471 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
472 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
473 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
474 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
475 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
476 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
477 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
478 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
479 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
480 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
481 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
482 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
483 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
484 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
485 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
486 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
487 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
488 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
489 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
490 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
491 /* 40 UNDEFINED */ hmR0VmxExitPause,
492 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
493 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
494 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
495 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
496 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
498 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
499 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
500 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
501 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
502 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
503 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
504 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
505 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
506 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
507 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
508 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
509 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
510 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
511 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
512 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
513 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
514 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
515 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
516};
517#endif /* HMVMX_USE_FUNCTION_TABLE */
518
519#ifdef VBOX_STRICT
520static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
521{
522 /* 0 */ "(Not Used)",
523 /* 1 */ "VMCALL executed in VMX root operation.",
524 /* 2 */ "VMCLEAR with invalid physical address.",
525 /* 3 */ "VMCLEAR with VMXON pointer.",
526 /* 4 */ "VMLAUNCH with non-clear VMCS.",
527 /* 5 */ "VMRESUME with non-launched VMCS.",
528 /* 6 */ "VMRESUME after VMXOFF",
529 /* 7 */ "VM-entry with invalid control fields.",
530 /* 8 */ "VM-entry with invalid host state fields.",
531 /* 9 */ "VMPTRLD with invalid physical address.",
532 /* 10 */ "VMPTRLD with VMXON pointer.",
533 /* 11 */ "VMPTRLD with incorrect revision identifier.",
534 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
535 /* 13 */ "VMWRITE to read-only VMCS component.",
536 /* 14 */ "(Not Used)",
537 /* 15 */ "VMXON executed in VMX root operation.",
538 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
539 /* 17 */ "VM-entry with non-launched executing VMCS.",
540 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
541 /* 19 */ "VMCALL with non-clear VMCS.",
542 /* 20 */ "VMCALL with invalid VM-exit control fields.",
543 /* 21 */ "(Not Used)",
544 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
545 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
546 /* 24 */ "VMCALL with invalid SMM-monitor features.",
547 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
548 /* 26 */ "VM-entry with events blocked by MOV SS.",
549 /* 27 */ "(Not Used)",
550 /* 28 */ "Invalid operand to INVEPT/INVVPID."
551};
552#endif /* VBOX_STRICT */
553
554
555
556/**
557 * Updates the VM's last error record.
558 *
559 * If there was a VMX instruction error, reads the error data from the VMCS and
560 * updates VCPU's last error record as well.
561 *
562 * @param pVM The cross context VM structure.
563 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
564 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
565 * VERR_VMX_INVALID_VMCS_FIELD.
566 * @param rc The error code.
567 */
568static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
569{
570 AssertPtr(pVM);
571 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
572 || rc == VERR_VMX_UNABLE_TO_START_VM)
573 {
574 AssertPtrReturnVoid(pVCpu);
575 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
576 }
577 pVM->hm.s.lLastError = rc;
578}
579
580
581/**
582 * Reads the VM-entry interruption-information field from the VMCS into the VMX
583 * transient structure.
584 *
585 * @returns VBox status code.
586 * @param pVmxTransient Pointer to the VMX transient structure.
587 *
588 * @remarks No-long-jump zone!!!
589 */
590DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
591{
592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
593 AssertRCReturn(rc, rc);
594 return VINF_SUCCESS;
595}
596
597
598#ifdef VBOX_STRICT
599/**
600 * Reads the VM-entry exception error code field from the VMCS into
601 * the VMX transient structure.
602 *
603 * @returns VBox status code.
604 * @param pVmxTransient Pointer to the VMX transient structure.
605 *
606 * @remarks No-long-jump zone!!!
607 */
608DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
611 AssertRCReturn(rc, rc);
612 return VINF_SUCCESS;
613}
614#endif /* VBOX_STRICT */
615
616
617#ifdef VBOX_STRICT
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#else
1516 RT_NOREF(pVCpu, uMsr);
1517#endif
1518 return false;
1519}
1520
1521
1522/**
1523 * Saves a set of guest MSRs back into the guest-CPU context.
1524 *
1525 * @param pVCpu The cross context virtual CPU structure.
1526 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1527 * out-of-sync. Make sure to update the required fields
1528 * before using them.
1529 *
1530 * @remarks No-long-jump zone!!!
1531 */
1532static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1533{
1534 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1535 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1536
1537 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1538 {
1539 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540#if HC_ARCH_BITS == 64
1541 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542 {
1543 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1544 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1545 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1546 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1547 }
1548#else
1549 NOREF(pMixedCtx);
1550#endif
1551 }
1552}
1553
1554
1555/**
1556 * Loads a set of guests MSRs to allow read/passthru to the guest.
1557 *
1558 * The name of this function is slightly confusing. This function does NOT
1559 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1560 * common prefix for functions dealing with "lazy restoration" of the shared
1561 * MSRs.
1562 *
1563 * @param pVCpu The cross context virtual CPU structure.
1564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1565 * out-of-sync. Make sure to update the required fields
1566 * before using them.
1567 *
1568 * @remarks No-long-jump zone!!!
1569 */
1570static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1571{
1572 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1573 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1574
1575 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1576#if HC_ARCH_BITS == 64
1577 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1578 {
1579 /*
1580 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1581 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1582 * we can skip a few MSR writes.
1583 *
1584 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1585 * guest MSR values in the guest-CPU context might be different to what's currently
1586 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1587 * CPU, see @bugref{8728}.
1588 */
1589 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1590 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1591 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1592 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1593 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1594 {
1595#ifdef VBOX_STRICT
1596 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1597 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1598 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1599 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1600#endif
1601 }
1602 else
1603 {
1604 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1605 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1606 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1607 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1608 }
1609 }
1610#else
1611 RT_NOREF(pMixedCtx);
1612#endif
1613 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1614}
1615
1616
1617/**
1618 * Performs lazy restoration of the set of host MSRs if they were previously
1619 * loaded with guest MSR values.
1620 *
1621 * @param pVCpu The cross context virtual CPU structure.
1622 *
1623 * @remarks No-long-jump zone!!!
1624 * @remarks The guest MSRs should have been saved back into the guest-CPU
1625 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1626 */
1627static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1628{
1629 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1630 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1631
1632 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1633 {
1634 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1635#if HC_ARCH_BITS == 64
1636 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1637 {
1638 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1639 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1640 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1641 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1642 }
1643#endif
1644 }
1645 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1646}
1647
1648
1649/**
1650 * Verifies that our cached values of the VMCS controls are all
1651 * consistent with what's actually present in the VMCS.
1652 *
1653 * @returns VBox status code.
1654 * @param pVCpu The cross context virtual CPU structure.
1655 */
1656static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1657{
1658 uint32_t u32Val;
1659 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1660 AssertRCReturn(rc, rc);
1661 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1662 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1663
1664 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1665 AssertRCReturn(rc, rc);
1666 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1667 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1668
1669 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1670 AssertRCReturn(rc, rc);
1671 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1672 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1673
1674 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1675 AssertRCReturn(rc, rc);
1676 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1677 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1678
1679 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1680 {
1681 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1682 AssertRCReturn(rc, rc);
1683 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1684 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1685 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1686 }
1687
1688 return VINF_SUCCESS;
1689}
1690
1691
1692#ifdef VBOX_STRICT
1693/**
1694 * Verifies that our cached host EFER value has not changed
1695 * since we cached it.
1696 *
1697 * @param pVCpu The cross context virtual CPU structure.
1698 */
1699static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1700{
1701 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1702
1703 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1704 {
1705 uint64_t u64Val;
1706 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1707 AssertRC(rc);
1708
1709 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1710 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1711 }
1712}
1713
1714
1715/**
1716 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1717 * VMCS are correct.
1718 *
1719 * @param pVCpu The cross context virtual CPU structure.
1720 */
1721static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1722{
1723 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1724
1725 /* Verify MSR counts in the VMCS are what we think it should be. */
1726 uint32_t cMsrs;
1727 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1728 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1729
1730 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1731 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1732
1733 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1734 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1735
1736 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1737 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1738 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1739 {
1740 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1741 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1742 pGuestMsr->u32Msr, cMsrs));
1743
1744 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1745 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1746 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1747
1748 /* Verify that the permissions are as expected in the MSR bitmap. */
1749 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1750 {
1751 VMXMSREXITREAD enmRead;
1752 VMXMSREXITWRITE enmWrite;
1753 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1754 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1755 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1756 {
1757 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1758 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1759 }
1760 else
1761 {
1762 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1763 pGuestMsr->u32Msr, cMsrs));
1764 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1765 pGuestMsr->u32Msr, cMsrs));
1766 }
1767 }
1768 }
1769}
1770#endif /* VBOX_STRICT */
1771
1772
1773/**
1774 * Flushes the TLB using EPT.
1775 *
1776 * @returns VBox status code.
1777 * @param pVCpu The cross context virtual CPU structure of the calling
1778 * EMT. Can be NULL depending on @a enmFlush.
1779 * @param enmFlush Type of flush.
1780 *
1781 * @remarks Caller is responsible for making sure this function is called only
1782 * when NestedPaging is supported and providing @a enmFlush that is
1783 * supported by the CPU.
1784 * @remarks Can be called with interrupts disabled.
1785 */
1786static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1787{
1788 uint64_t au64Descriptor[2];
1789 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1790 au64Descriptor[0] = 0;
1791 else
1792 {
1793 Assert(pVCpu);
1794 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1795 }
1796 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1797
1798 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1799 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1800 rc));
1801 if ( RT_SUCCESS(rc)
1802 && pVCpu)
1803 {
1804 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1805 }
1806}
1807
1808
1809/**
1810 * Flushes the TLB using VPID.
1811 *
1812 * @returns VBox status code.
1813 * @param pVM The cross context VM structure.
1814 * @param pVCpu The cross context virtual CPU structure of the calling
1815 * EMT. Can be NULL depending on @a enmFlush.
1816 * @param enmFlush Type of flush.
1817 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1818 * on @a enmFlush).
1819 *
1820 * @remarks Can be called with interrupts disabled.
1821 */
1822static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1823{
1824 NOREF(pVM);
1825 AssertPtr(pVM);
1826 Assert(pVM->hm.s.vmx.fVpid);
1827
1828 uint64_t au64Descriptor[2];
1829 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1830 {
1831 au64Descriptor[0] = 0;
1832 au64Descriptor[1] = 0;
1833 }
1834 else
1835 {
1836 AssertPtr(pVCpu);
1837 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1838 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1839 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1840 au64Descriptor[1] = GCPtr;
1841 }
1842
1843 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1844 AssertMsg(rc == VINF_SUCCESS,
1845 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1846 if ( RT_SUCCESS(rc)
1847 && pVCpu)
1848 {
1849 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1850 }
1851}
1852
1853
1854/**
1855 * Invalidates a guest page by guest virtual address. Only relevant for
1856 * EPT/VPID, otherwise there is nothing really to invalidate.
1857 *
1858 * @returns VBox status code.
1859 * @param pVM The cross context VM structure.
1860 * @param pVCpu The cross context virtual CPU structure.
1861 * @param GCVirt Guest virtual address of the page to invalidate.
1862 */
1863VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1864{
1865 AssertPtr(pVM);
1866 AssertPtr(pVCpu);
1867 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1868
1869 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1870 if (!fFlushPending)
1871 {
1872 /*
1873 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1874 * See @bugref{6043} and @bugref{6177}.
1875 *
1876 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1877 * function maybe called in a loop with individual addresses.
1878 */
1879 if (pVM->hm.s.vmx.fVpid)
1880 {
1881 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1882 {
1883 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1884 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1885 }
1886 else
1887 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1888 }
1889 else if (pVM->hm.s.fNestedPaging)
1890 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1891 }
1892
1893 return VINF_SUCCESS;
1894}
1895
1896
1897/**
1898 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1899 * otherwise there is nothing really to invalidate.
1900 *
1901 * @returns VBox status code.
1902 * @param pVM The cross context VM structure.
1903 * @param pVCpu The cross context virtual CPU structure.
1904 * @param GCPhys Guest physical address of the page to invalidate.
1905 */
1906VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1907{
1908 NOREF(pVM); NOREF(GCPhys);
1909 LogFlowFunc(("%RGp\n", GCPhys));
1910
1911 /*
1912 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1913 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1914 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1915 */
1916 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1917 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1918 return VINF_SUCCESS;
1919}
1920
1921
1922/**
1923 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1924 * case where neither EPT nor VPID is supported by the CPU.
1925 *
1926 * @param pVM The cross context VM structure.
1927 * @param pVCpu The cross context virtual CPU structure.
1928 * @param pCpu Pointer to the global HM struct.
1929 *
1930 * @remarks Called with interrupts disabled.
1931 */
1932static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1933{
1934 AssertPtr(pVCpu);
1935 AssertPtr(pCpu);
1936 NOREF(pVM);
1937
1938 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1939
1940 Assert(pCpu->idCpu != NIL_RTCPUID);
1941 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1942 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1943 pVCpu->hm.s.fForceTLBFlush = false;
1944 return;
1945}
1946
1947
1948/**
1949 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1950 *
1951 * @param pVM The cross context VM structure.
1952 * @param pVCpu The cross context virtual CPU structure.
1953 * @param pCpu Pointer to the global HM CPU struct.
1954 * @remarks All references to "ASID" in this function pertains to "VPID" in
1955 * Intel's nomenclature. The reason is, to avoid confusion in compare
1956 * statements since the host-CPU copies are named "ASID".
1957 *
1958 * @remarks Called with interrupts disabled.
1959 */
1960static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1961{
1962#ifdef VBOX_WITH_STATISTICS
1963 bool fTlbFlushed = false;
1964# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1965# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1966 if (!fTlbFlushed) \
1967 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1968 } while (0)
1969#else
1970# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1971# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1972#endif
1973
1974 AssertPtr(pVM);
1975 AssertPtr(pCpu);
1976 AssertPtr(pVCpu);
1977 Assert(pCpu->idCpu != NIL_RTCPUID);
1978
1979 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1980 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1981 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1982
1983 /*
1984 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1985 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1986 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1987 */
1988 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1989 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1990 {
1991 ++pCpu->uCurrentAsid;
1992 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1993 {
1994 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1995 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1996 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1997 }
1998
1999 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2000 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2001 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2002
2003 /*
2004 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2005 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2006 */
2007 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2008 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2009 HMVMX_SET_TAGGED_TLB_FLUSHED();
2010 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2011 }
2012
2013 /* Check for explicit TLB flushes. */
2014 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2015 {
2016 /*
2017 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2018 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2019 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2020 * but not guest-physical mappings.
2021 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2022 */
2023 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2024 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2025 HMVMX_SET_TAGGED_TLB_FLUSHED();
2026 }
2027
2028 pVCpu->hm.s.fForceTLBFlush = false;
2029 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2030
2031 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2032 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2033 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2034 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2035 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2036 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2037 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2038 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2039 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2040
2041 /* Update VMCS with the VPID. */
2042 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2043 AssertRC(rc);
2044
2045#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2046}
2047
2048
2049/**
2050 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2051 *
2052 * @returns VBox status code.
2053 * @param pVM The cross context VM structure.
2054 * @param pVCpu The cross context virtual CPU structure.
2055 * @param pCpu Pointer to the global HM CPU struct.
2056 *
2057 * @remarks Called with interrupts disabled.
2058 */
2059static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2060{
2061 AssertPtr(pVM);
2062 AssertPtr(pVCpu);
2063 AssertPtr(pCpu);
2064 Assert(pCpu->idCpu != NIL_RTCPUID);
2065 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2066 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2067
2068 /*
2069 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2070 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2071 */
2072 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2073 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2074 {
2075 pVCpu->hm.s.fForceTLBFlush = true;
2076 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2077 }
2078
2079 /* Check for explicit TLB flushes. */
2080 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2081 {
2082 pVCpu->hm.s.fForceTLBFlush = true;
2083 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2084 }
2085
2086 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2087 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2088
2089 if (pVCpu->hm.s.fForceTLBFlush)
2090 {
2091 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2092 pVCpu->hm.s.fForceTLBFlush = false;
2093 }
2094}
2095
2096
2097/**
2098 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2099 *
2100 * @returns VBox status code.
2101 * @param pVM The cross context VM structure.
2102 * @param pVCpu The cross context virtual CPU structure.
2103 * @param pCpu Pointer to the global HM CPU struct.
2104 *
2105 * @remarks Called with interrupts disabled.
2106 */
2107static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2108{
2109 AssertPtr(pVM);
2110 AssertPtr(pVCpu);
2111 AssertPtr(pCpu);
2112 Assert(pCpu->idCpu != NIL_RTCPUID);
2113 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2114 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2115
2116 /*
2117 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2118 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2119 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2120 */
2121 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2122 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2123 {
2124 pVCpu->hm.s.fForceTLBFlush = true;
2125 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2126 }
2127
2128 /* Check for explicit TLB flushes. */
2129 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2130 {
2131 /*
2132 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2133 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2134 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2135 */
2136 pVCpu->hm.s.fForceTLBFlush = true;
2137 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2138 }
2139
2140 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2141 if (pVCpu->hm.s.fForceTLBFlush)
2142 {
2143 ++pCpu->uCurrentAsid;
2144 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2145 {
2146 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2147 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2148 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2149 }
2150
2151 pVCpu->hm.s.fForceTLBFlush = false;
2152 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2153 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2154 if (pCpu->fFlushAsidBeforeUse)
2155 {
2156 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2157 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2158 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2159 {
2160 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2161 pCpu->fFlushAsidBeforeUse = false;
2162 }
2163 else
2164 {
2165 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2166 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2167 }
2168 }
2169 }
2170
2171 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2172 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2173 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2174 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2175 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2176 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2177 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2178
2179 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2180 AssertRC(rc);
2181}
2182
2183
2184/**
2185 * Flushes the guest TLB entry based on CPU capabilities.
2186 *
2187 * @param pVCpu The cross context virtual CPU structure.
2188 * @param pCpu Pointer to the global HM CPU struct.
2189 */
2190DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2191{
2192#ifdef HMVMX_ALWAYS_FLUSH_TLB
2193 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2194#endif
2195 PVM pVM = pVCpu->CTX_SUFF(pVM);
2196 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2197 {
2198 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2199 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2200 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2201 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2202 default:
2203 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2204 break;
2205 }
2206
2207 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2208}
2209
2210
2211/**
2212 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2213 * TLB entries from the host TLB before VM-entry.
2214 *
2215 * @returns VBox status code.
2216 * @param pVM The cross context VM structure.
2217 */
2218static int hmR0VmxSetupTaggedTlb(PVM pVM)
2219{
2220 /*
2221 * Determine optimal flush type for Nested Paging.
2222 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2223 * guest execution (see hmR3InitFinalizeR0()).
2224 */
2225 if (pVM->hm.s.fNestedPaging)
2226 {
2227 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2228 {
2229 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2230 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2231 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2232 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2233 else
2234 {
2235 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2236 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2237 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2238 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2239 }
2240
2241 /* Make sure the write-back cacheable memory type for EPT is supported. */
2242 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2243 {
2244 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2245 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2246 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2247 }
2248
2249 /* EPT requires a page-walk length of 4. */
2250 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2251 {
2252 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2253 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2254 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2255 }
2256 }
2257 else
2258 {
2259 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2260 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2261 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2262 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2263 }
2264 }
2265
2266 /*
2267 * Determine optimal flush type for VPID.
2268 */
2269 if (pVM->hm.s.vmx.fVpid)
2270 {
2271 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2272 {
2273 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2274 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2275 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2276 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2277 else
2278 {
2279 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2280 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2281 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2282 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2283 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2284 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2285 pVM->hm.s.vmx.fVpid = false;
2286 }
2287 }
2288 else
2289 {
2290 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2291 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2292 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2293 pVM->hm.s.vmx.fVpid = false;
2294 }
2295 }
2296
2297 /*
2298 * Setup the handler for flushing tagged-TLBs.
2299 */
2300 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2301 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2302 else if (pVM->hm.s.fNestedPaging)
2303 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2304 else if (pVM->hm.s.vmx.fVpid)
2305 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2306 else
2307 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2308 return VINF_SUCCESS;
2309}
2310
2311
2312/**
2313 * Sets up pin-based VM-execution controls in the VMCS.
2314 *
2315 * @returns VBox status code.
2316 * @param pVM The cross context VM structure.
2317 * @param pVCpu The cross context virtual CPU structure.
2318 */
2319static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2320{
2321 AssertPtr(pVM);
2322 AssertPtr(pVCpu);
2323
2324 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2325 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2326
2327 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2328 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2329
2330 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2331 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2332
2333 /* Enable the VMX preemption timer. */
2334 if (pVM->hm.s.vmx.fUsePreemptTimer)
2335 {
2336 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2337 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2338 }
2339
2340#if 0
2341 /* Enable posted-interrupt processing. */
2342 if (pVM->hm.s.fPostedIntrs)
2343 {
2344 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2345 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2346 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2347 }
2348#endif
2349
2350 if ((val & zap) != val)
2351 {
2352 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2353 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2354 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2355 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2356 }
2357
2358 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2359 AssertRCReturn(rc, rc);
2360
2361 pVCpu->hm.s.vmx.u32PinCtls = val;
2362 return rc;
2363}
2364
2365
2366/**
2367 * Sets up processor-based VM-execution controls in the VMCS.
2368 *
2369 * @returns VBox status code.
2370 * @param pVM The cross context VM structure.
2371 * @param pVCpu The cross context virtual CPU structure.
2372 */
2373static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2374{
2375 AssertPtr(pVM);
2376 AssertPtr(pVCpu);
2377
2378 int rc = VERR_INTERNAL_ERROR_5;
2379 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2380 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2381
2382 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2383 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2384 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2385 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2386 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2387 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2388 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2389
2390 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2391 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2392 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2393 {
2394 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2395 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2396 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2397 }
2398
2399 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2400 if (!pVM->hm.s.fNestedPaging)
2401 {
2402 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2403 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2404 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2405 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2406 }
2407
2408 /* Use TPR shadowing if supported by the CPU. */
2409 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2410 {
2411 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2412 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2413 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2414 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2415 AssertRCReturn(rc, rc);
2416
2417 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2418 /* CR8 writes cause a VM-exit based on TPR threshold. */
2419 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2420 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2421 }
2422 else
2423 {
2424 /*
2425 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2426 * Set this control only for 64-bit guests.
2427 */
2428 if (pVM->hm.s.fAllow64BitGuests)
2429 {
2430 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2431 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2432 }
2433 }
2434
2435 /* Use MSR-bitmaps if supported by the CPU. */
2436 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2437 {
2438 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2439
2440 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2441 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2442 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2443 AssertRCReturn(rc, rc);
2444
2445 /*
2446 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2447 * automatically using dedicated fields in the VMCS.
2448 */
2449 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454
2455#if HC_ARCH_BITS == 64
2456 /*
2457 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2458 */
2459 if (pVM->hm.s.fAllow64BitGuests)
2460 {
2461 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2462 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2463 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2464 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2465 }
2466#endif
2467 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2468 }
2469
2470 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2471 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2472 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2473
2474 if ((val & zap) != val)
2475 {
2476 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2477 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2478 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2479 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2480 }
2481
2482 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2483 AssertRCReturn(rc, rc);
2484
2485 pVCpu->hm.s.vmx.u32ProcCtls = val;
2486
2487 /*
2488 * Secondary processor-based VM-execution controls.
2489 */
2490 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2491 {
2492 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2493 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2494
2495 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2496 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2497
2498 if (pVM->hm.s.fNestedPaging)
2499 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2500 else
2501 {
2502 /*
2503 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2504 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2505 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2506 */
2507 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2509 }
2510
2511 if (pVM->hm.s.vmx.fVpid)
2512 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2513
2514 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2515 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2516
2517#if 0
2518 if (pVM->hm.s.fVirtApicRegs)
2519 {
2520 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2521 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2522
2523 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2524 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2525 }
2526#endif
2527
2528 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2529 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2530 * done dynamically. */
2531 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2532 {
2533 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2534 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2535 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2536 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2537 AssertRCReturn(rc, rc);
2538 }
2539
2540 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2541 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2542
2543 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2544 && pVM->hm.s.vmx.cPleGapTicks
2545 && pVM->hm.s.vmx.cPleWindowTicks)
2546 {
2547 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2548
2549 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2550 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2551 AssertRCReturn(rc, rc);
2552 }
2553
2554 if ((val & zap) != val)
2555 {
2556 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2557 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2558 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2559 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2560 }
2561
2562 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2563 AssertRCReturn(rc, rc);
2564
2565 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2566 }
2567 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2568 {
2569 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2570 "available\n"));
2571 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2572 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2573 }
2574
2575 return VINF_SUCCESS;
2576}
2577
2578
2579/**
2580 * Sets up miscellaneous (everything other than Pin & Processor-based
2581 * VM-execution) control fields in the VMCS.
2582 *
2583 * @returns VBox status code.
2584 * @param pVM The cross context VM structure.
2585 * @param pVCpu The cross context virtual CPU structure.
2586 */
2587static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2588{
2589 NOREF(pVM);
2590 AssertPtr(pVM);
2591 AssertPtr(pVCpu);
2592
2593 int rc = VERR_GENERAL_FAILURE;
2594
2595 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2596#if 0
2597 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2598 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2599 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2600
2601 /*
2602 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2603 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2604 * We thus use the exception bitmap to control it rather than use both.
2605 */
2606 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2607 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2608
2609 /** @todo Explore possibility of using IO-bitmaps. */
2610 /* All IO & IOIO instructions cause VM-exits. */
2611 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2612 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2613
2614 /* Initialize the MSR-bitmap area. */
2615 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2616 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2617 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2618 AssertRCReturn(rc, rc);
2619#endif
2620
2621 /* Setup MSR auto-load/store area. */
2622 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2623 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2624 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2625 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2626 AssertRCReturn(rc, rc);
2627
2628 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2629 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2630 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2631 AssertRCReturn(rc, rc);
2632
2633 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2634 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2635 AssertRCReturn(rc, rc);
2636
2637 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2638#if 0
2639 /* Setup debug controls */
2640 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2641 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2642 AssertRCReturn(rc, rc);
2643#endif
2644
2645 return rc;
2646}
2647
2648
2649/**
2650 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2651 *
2652 * We shall setup those exception intercepts that don't change during the
2653 * lifetime of the VM here. The rest are done dynamically while loading the
2654 * guest state.
2655 *
2656 * @returns VBox status code.
2657 * @param pVM The cross context VM structure.
2658 * @param pVCpu The cross context virtual CPU structure.
2659 */
2660static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2661{
2662 AssertPtr(pVM);
2663 AssertPtr(pVCpu);
2664
2665 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2666
2667 uint32_t u32XcptBitmap = 0;
2668
2669 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2670 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2671
2672 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2673 and writes, and because recursive #DBs can cause the CPU hang, we must always
2674 intercept #DB. */
2675 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2676
2677 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2678 if (!pVM->hm.s.fNestedPaging)
2679 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2680
2681 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2682 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2683 AssertRCReturn(rc, rc);
2684 return rc;
2685}
2686
2687
2688/**
2689 * Sets up the initial guest-state mask. The guest-state mask is consulted
2690 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2691 * for the nested virtualization case (as it would cause a VM-exit).
2692 *
2693 * @param pVCpu The cross context virtual CPU structure.
2694 */
2695static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2696{
2697 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2698 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2699 return VINF_SUCCESS;
2700}
2701
2702
2703/**
2704 * Does per-VM VT-x initialization.
2705 *
2706 * @returns VBox status code.
2707 * @param pVM The cross context VM structure.
2708 */
2709VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2710{
2711 LogFlowFunc(("pVM=%p\n", pVM));
2712
2713 int rc = hmR0VmxStructsAlloc(pVM);
2714 if (RT_FAILURE(rc))
2715 {
2716 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2717 return rc;
2718 }
2719
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Does per-VM VT-x termination.
2726 *
2727 * @returns VBox status code.
2728 * @param pVM The cross context VM structure.
2729 */
2730VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2731{
2732 LogFlowFunc(("pVM=%p\n", pVM));
2733
2734#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2735 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2736 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2737#endif
2738 hmR0VmxStructsFree(pVM);
2739 return VINF_SUCCESS;
2740}
2741
2742
2743/**
2744 * Sets up the VM for execution under VT-x.
2745 * This function is only called once per-VM during initialization.
2746 *
2747 * @returns VBox status code.
2748 * @param pVM The cross context VM structure.
2749 */
2750VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2751{
2752 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2753 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2754
2755 LogFlowFunc(("pVM=%p\n", pVM));
2756
2757 /*
2758 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2759 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2760 */
2761 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2762 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2763 || !pVM->hm.s.vmx.pRealModeTSS))
2764 {
2765 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2766 return VERR_INTERNAL_ERROR;
2767 }
2768
2769 /* Initialize these always, see hmR3InitFinalizeR0().*/
2770 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2771 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2772
2773 /* Setup the tagged-TLB flush handlers. */
2774 int rc = hmR0VmxSetupTaggedTlb(pVM);
2775 if (RT_FAILURE(rc))
2776 {
2777 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2778 return rc;
2779 }
2780
2781 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2782 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2783#if HC_ARCH_BITS == 64
2784 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2785 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2786 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2787 {
2788 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2789 }
2790#endif
2791
2792 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2793 RTCCUINTREG uHostCR4 = ASMGetCR4();
2794 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2795 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2796
2797 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2798 {
2799 PVMCPU pVCpu = &pVM->aCpus[i];
2800 AssertPtr(pVCpu);
2801 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2802
2803 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2804 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2805
2806 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2807 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2808 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2809
2810 /* Set revision dword at the beginning of the VMCS structure. */
2811 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2812
2813 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2814 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2815 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818 /* Load this VMCS as the current VMCS. */
2819 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2820 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2821 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2822
2823 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2824 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2825 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2826
2827 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2828 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2829 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2830
2831 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2836 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2837 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2838
2839 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2840 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2841 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2842
2843#if HC_ARCH_BITS == 32
2844 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2845 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2846 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2847#endif
2848
2849 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2850 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2851 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2852 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2853
2854 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2855
2856 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2857 }
2858
2859 return VINF_SUCCESS;
2860}
2861
2862
2863/**
2864 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2865 * the VMCS.
2866 *
2867 * @returns VBox status code.
2868 * @param pVM The cross context VM structure.
2869 * @param pVCpu The cross context virtual CPU structure.
2870 */
2871DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2872{
2873 NOREF(pVM); NOREF(pVCpu);
2874
2875 RTCCUINTREG uReg = ASMGetCR0();
2876 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2877 AssertRCReturn(rc, rc);
2878
2879 uReg = ASMGetCR3();
2880 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2881 AssertRCReturn(rc, rc);
2882
2883 uReg = ASMGetCR4();
2884 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2885 AssertRCReturn(rc, rc);
2886 return rc;
2887}
2888
2889
2890#if HC_ARCH_BITS == 64
2891/**
2892 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2893 * requirements. See hmR0VmxSaveHostSegmentRegs().
2894 */
2895# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2896 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2897 { \
2898 bool fValidSelector = true; \
2899 if ((selValue) & X86_SEL_LDT) \
2900 { \
2901 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2902 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2903 } \
2904 if (fValidSelector) \
2905 { \
2906 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2907 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2908 } \
2909 (selValue) = 0; \
2910 }
2911#endif
2912
2913
2914/**
2915 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2916 * the host-state area in the VMCS.
2917 *
2918 * @returns VBox status code.
2919 * @param pVM The cross context VM structure.
2920 * @param pVCpu The cross context virtual CPU structure.
2921 */
2922DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2923{
2924 int rc = VERR_INTERNAL_ERROR_5;
2925
2926#if HC_ARCH_BITS == 64
2927 /*
2928 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2929 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2930 *
2931 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2932 * Was observed booting Solaris10u10 32-bit guest.
2933 */
2934 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2935 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2936 {
2937 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2938 pVCpu->idCpu));
2939 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2940 }
2941 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2942#else
2943 RT_NOREF(pVCpu);
2944#endif
2945
2946 /*
2947 * Host DS, ES, FS and GS segment registers.
2948 */
2949#if HC_ARCH_BITS == 64
2950 RTSEL uSelDS = ASMGetDS();
2951 RTSEL uSelES = ASMGetES();
2952 RTSEL uSelFS = ASMGetFS();
2953 RTSEL uSelGS = ASMGetGS();
2954#else
2955 RTSEL uSelDS = 0;
2956 RTSEL uSelES = 0;
2957 RTSEL uSelFS = 0;
2958 RTSEL uSelGS = 0;
2959#endif
2960
2961 /*
2962 * Host CS and SS segment registers.
2963 */
2964 RTSEL uSelCS = ASMGetCS();
2965 RTSEL uSelSS = ASMGetSS();
2966
2967 /*
2968 * Host TR segment register.
2969 */
2970 RTSEL uSelTR = ASMGetTR();
2971
2972#if HC_ARCH_BITS == 64
2973 /*
2974 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2975 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2976 */
2977 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2978 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2979 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2980 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2981# undef VMXLOCAL_ADJUST_HOST_SEG
2982#endif
2983
2984 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2985 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2986 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2987 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2988 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2989 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2990 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2991 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2992 Assert(uSelCS);
2993 Assert(uSelTR);
2994
2995 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2996#if 0
2997 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2998 Assert(uSelSS != 0);
2999#endif
3000
3001 /* Write these host selector fields into the host-state area in the VMCS. */
3002 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3003 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3004#if HC_ARCH_BITS == 64
3005 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3006 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3007 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3008 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3009#else
3010 NOREF(uSelDS);
3011 NOREF(uSelES);
3012 NOREF(uSelFS);
3013 NOREF(uSelGS);
3014#endif
3015 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3016 AssertRCReturn(rc, rc);
3017
3018 /*
3019 * Host GDTR and IDTR.
3020 */
3021 RTGDTR Gdtr;
3022 RTIDTR Idtr;
3023 RT_ZERO(Gdtr);
3024 RT_ZERO(Idtr);
3025 ASMGetGDTR(&Gdtr);
3026 ASMGetIDTR(&Idtr);
3027 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3028 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3029 AssertRCReturn(rc, rc);
3030
3031#if HC_ARCH_BITS == 64
3032 /*
3033 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3034 * maximum limit (0xffff) on every VM-exit.
3035 */
3036 if (Gdtr.cbGdt != 0xffff)
3037 {
3038 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3039 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3040 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3041 }
3042
3043 /*
3044 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3045 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3046 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3047 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3048 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3049 * hosts where we are pretty sure it won't cause trouble.
3050 */
3051# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3052 if (Idtr.cbIdt < 0x0fff)
3053# else
3054 if (Idtr.cbIdt != 0xffff)
3055# endif
3056 {
3057 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3058 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3059 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3060 }
3061#endif
3062
3063 /*
3064 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3065 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3066 */
3067 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3068 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3069 VERR_VMX_INVALID_HOST_STATE);
3070
3071 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3072#if HC_ARCH_BITS == 64
3073 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3074
3075 /*
3076 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3077 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3078 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3079 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3080 *
3081 * [1] See Intel spec. 3.5 "System Descriptor Types".
3082 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3083 */
3084 Assert(pDesc->System.u4Type == 11);
3085 if ( pDesc->System.u16LimitLow != 0x67
3086 || pDesc->System.u4LimitHigh)
3087 {
3088 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3089 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3090 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3091 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3092 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3093
3094 /* Store the GDTR here as we need it while restoring TR. */
3095 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3096 }
3097#else
3098 NOREF(pVM);
3099 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3100#endif
3101 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3102 AssertRCReturn(rc, rc);
3103
3104 /*
3105 * Host FS base and GS base.
3106 */
3107#if HC_ARCH_BITS == 64
3108 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3109 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3110 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3111 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3112 AssertRCReturn(rc, rc);
3113
3114 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3115 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3116 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3117 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3118 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3119#endif
3120 return rc;
3121}
3122
3123
3124/**
3125 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3126 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3127 * the host after every successful VM-exit.
3128 *
3129 * @returns VBox status code.
3130 * @param pVM The cross context VM structure.
3131 * @param pVCpu The cross context virtual CPU structure.
3132 *
3133 * @remarks No-long-jump zone!!!
3134 */
3135DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3136{
3137 NOREF(pVM);
3138
3139 AssertPtr(pVCpu);
3140 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3141
3142 /*
3143 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3144 * rather than swapping them on every VM-entry.
3145 */
3146 hmR0VmxLazySaveHostMsrs(pVCpu);
3147
3148 /*
3149 * Host Sysenter MSRs.
3150 */
3151 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3152#if HC_ARCH_BITS == 32
3153 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3154 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3155#else
3156 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3157 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3158#endif
3159 AssertRCReturn(rc, rc);
3160
3161 /*
3162 * Host EFER MSR.
3163 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3164 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3165 */
3166 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3167 {
3168 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3169 AssertRCReturn(rc, rc);
3170 }
3171
3172 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3173 * hmR0VmxLoadGuestExitCtls() !! */
3174
3175 return rc;
3176}
3177
3178
3179/**
3180 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3181 *
3182 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3183 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3184 * hmR0VMxLoadGuestEntryCtls().
3185 *
3186 * @returns true if we need to load guest EFER, false otherwise.
3187 * @param pVCpu The cross context virtual CPU structure.
3188 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3189 * out-of-sync. Make sure to update the required fields
3190 * before using them.
3191 *
3192 * @remarks Requires EFER, CR4.
3193 * @remarks No-long-jump zone!!!
3194 */
3195static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3196{
3197#ifdef HMVMX_ALWAYS_SWAP_EFER
3198 return true;
3199#endif
3200
3201#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3202 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3203 if (CPUMIsGuestInLongMode(pVCpu))
3204 return false;
3205#endif
3206
3207 PVM pVM = pVCpu->CTX_SUFF(pVM);
3208 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3209 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3210
3211 /*
3212 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3213 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3214 */
3215 if ( CPUMIsGuestInLongMode(pVCpu)
3216 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3217 {
3218 return true;
3219 }
3220
3221 /*
3222 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3223 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3224 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3225 */
3226 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3227 && (pMixedCtx->cr0 & X86_CR0_PG)
3228 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3229 {
3230 /* Assert that host is PAE capable. */
3231 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3232 return true;
3233 }
3234
3235 /** @todo Check the latest Intel spec. for any other bits,
3236 * like SMEP/SMAP? */
3237 return false;
3238}
3239
3240
3241/**
3242 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3243 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3244 * controls".
3245 *
3246 * @returns VBox status code.
3247 * @param pVCpu The cross context virtual CPU structure.
3248 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3249 * out-of-sync. Make sure to update the required fields
3250 * before using them.
3251 *
3252 * @remarks Requires EFER.
3253 * @remarks No-long-jump zone!!!
3254 */
3255DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3256{
3257 int rc = VINF_SUCCESS;
3258 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3259 {
3260 PVM pVM = pVCpu->CTX_SUFF(pVM);
3261 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3262 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3263
3264 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3265 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3266
3267 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3268 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3269 {
3270 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3271 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3272 }
3273 else
3274 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3275
3276 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3277 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3278 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3279 {
3280 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3281 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3282 }
3283
3284 /*
3285 * The following should -not- be set (since we're not in SMM mode):
3286 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3287 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3288 */
3289
3290 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3291 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3292
3293 if ((val & zap) != val)
3294 {
3295 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3296 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3297 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3298 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3299 }
3300
3301 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3302 AssertRCReturn(rc, rc);
3303
3304 pVCpu->hm.s.vmx.u32EntryCtls = val;
3305 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3306 }
3307 return rc;
3308}
3309
3310
3311/**
3312 * Sets up the VM-exit controls in the VMCS.
3313 *
3314 * @returns VBox status code.
3315 * @param pVCpu The cross context virtual CPU structure.
3316 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3317 * out-of-sync. Make sure to update the required fields
3318 * before using them.
3319 *
3320 * @remarks Requires EFER.
3321 */
3322DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3323{
3324 NOREF(pMixedCtx);
3325
3326 int rc = VINF_SUCCESS;
3327 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3328 {
3329 PVM pVM = pVCpu->CTX_SUFF(pVM);
3330 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3331 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3332
3333 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3334 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3335
3336 /*
3337 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3338 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3339 */
3340#if HC_ARCH_BITS == 64
3341 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3342 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3343#else
3344 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3345 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3346 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3347 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3348 {
3349 /* The switcher returns to long mode, EFER is managed by the switcher. */
3350 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3351 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3352 }
3353 else
3354 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3355#endif
3356
3357 /* If the newer VMCS fields for managing EFER exists, use it. */
3358 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3359 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3360 {
3361 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3362 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3363 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3364 }
3365
3366 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3367 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3368
3369 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3370 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3371 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3372
3373 if ( pVM->hm.s.vmx.fUsePreemptTimer
3374 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3375 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3376
3377 if ((val & zap) != val)
3378 {
3379 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3380 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3381 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3382 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3383 }
3384
3385 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3386 AssertRCReturn(rc, rc);
3387
3388 pVCpu->hm.s.vmx.u32ExitCtls = val;
3389 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3390 }
3391 return rc;
3392}
3393
3394
3395/**
3396 * Sets the TPR threshold in the VMCS.
3397 *
3398 * @returns VBox status code.
3399 * @param pVCpu The cross context virtual CPU structure.
3400 * @param u32TprThreshold The TPR threshold (task-priority class only).
3401 */
3402DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3403{
3404 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3405 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3406 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3407}
3408
3409
3410/**
3411 * Loads the guest APIC and related state.
3412 *
3413 * @returns VBox status code.
3414 * @param pVCpu The cross context virtual CPU structure.
3415 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3416 * out-of-sync. Make sure to update the required fields
3417 * before using them.
3418 *
3419 * @remarks No-long-jump zone!!!
3420 */
3421DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3422{
3423 NOREF(pMixedCtx);
3424
3425 int rc = VINF_SUCCESS;
3426 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3427 {
3428 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3429 && APICIsEnabled(pVCpu))
3430 {
3431 /*
3432 * Setup TPR shadowing.
3433 */
3434 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3435 {
3436 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3437
3438 bool fPendingIntr = false;
3439 uint8_t u8Tpr = 0;
3440 uint8_t u8PendingIntr = 0;
3441 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3442 AssertRCReturn(rc, rc);
3443
3444 /*
3445 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3446 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3447 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3448 */
3449 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3450 uint32_t u32TprThreshold = 0;
3451 if (fPendingIntr)
3452 {
3453 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3454 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3455 const uint8_t u8TprPriority = u8Tpr >> 4;
3456 if (u8PendingPriority <= u8TprPriority)
3457 u32TprThreshold = u8PendingPriority;
3458 }
3459
3460 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3461 AssertRCReturn(rc, rc);
3462 }
3463 }
3464 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3465 }
3466
3467 return rc;
3468}
3469
3470
3471/**
3472 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3473 *
3474 * @returns Guest's interruptibility-state.
3475 * @param pVCpu The cross context virtual CPU structure.
3476 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3477 * out-of-sync. Make sure to update the required fields
3478 * before using them.
3479 *
3480 * @remarks No-long-jump zone!!!
3481 */
3482DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3483{
3484 /*
3485 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3486 */
3487 uint32_t uIntrState = 0;
3488 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3489 {
3490 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3491 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3492 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3493 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3494 {
3495 if (pMixedCtx->eflags.Bits.u1IF)
3496 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3497 else
3498 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3499 }
3500 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3501 {
3502 /*
3503 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3504 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3505 */
3506 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3507 }
3508 }
3509
3510 /*
3511 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3512 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3513 * setting this would block host-NMIs and IRET will not clear the blocking.
3514 *
3515 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3516 */
3517 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3518 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3519 {
3520 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3521 }
3522
3523 return uIntrState;
3524}
3525
3526
3527/**
3528 * Loads the guest's interruptibility-state into the guest-state area in the
3529 * VMCS.
3530 *
3531 * @returns VBox status code.
3532 * @param pVCpu The cross context virtual CPU structure.
3533 * @param uIntrState The interruptibility-state to set.
3534 */
3535static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3536{
3537 NOREF(pVCpu);
3538 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3539 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3540 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3541 AssertRC(rc);
3542 return rc;
3543}
3544
3545
3546/**
3547 * Loads the exception intercepts required for guest execution in the VMCS.
3548 *
3549 * @returns VBox status code.
3550 * @param pVCpu The cross context virtual CPU structure.
3551 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3552 * out-of-sync. Make sure to update the required fields
3553 * before using them.
3554 */
3555static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3556{
3557 NOREF(pMixedCtx);
3558 int rc = VINF_SUCCESS;
3559 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3560 {
3561 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3562 if (pVCpu->hm.s.fGIMTrapXcptUD)
3563 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3564#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3565 else
3566 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3567#endif
3568
3569 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3570 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3571
3572 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3573 AssertRCReturn(rc, rc);
3574
3575 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3576 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3577 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3578 }
3579 return rc;
3580}
3581
3582
3583/**
3584 * Loads the guest's RIP into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu The cross context virtual CPU structure.
3588 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3589 * out-of-sync. Make sure to update the required fields
3590 * before using them.
3591 *
3592 * @remarks No-long-jump zone!!!
3593 */
3594static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3595{
3596 int rc = VINF_SUCCESS;
3597 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3598 {
3599 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3600 AssertRCReturn(rc, rc);
3601
3602 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3603 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3604 HMCPU_CF_VALUE(pVCpu)));
3605 }
3606 return rc;
3607}
3608
3609
3610/**
3611 * Loads the guest's RSP into the guest-state area in the VMCS.
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu The cross context virtual CPU structure.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 int rc = VINF_SUCCESS;
3624 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3625 {
3626 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3627 AssertRCReturn(rc, rc);
3628
3629 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3630 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3631 }
3632 return rc;
3633}
3634
3635
3636/**
3637 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3638 *
3639 * @returns VBox status code.
3640 * @param pVCpu The cross context virtual CPU structure.
3641 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3642 * out-of-sync. Make sure to update the required fields
3643 * before using them.
3644 *
3645 * @remarks No-long-jump zone!!!
3646 */
3647static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3648{
3649 int rc = VINF_SUCCESS;
3650 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3651 {
3652 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3653 Let us assert it as such and use 32-bit VMWRITE. */
3654 Assert(!(pMixedCtx->rflags.u64 >> 32));
3655 X86EFLAGS Eflags = pMixedCtx->eflags;
3656 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3657 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3658 * These will never be cleared/set, unless some other part of the VMM
3659 * code is buggy - in which case we're better of finding and fixing
3660 * those bugs than hiding them. */
3661 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3662 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3663 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3664 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3665
3666 /*
3667 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3668 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3669 */
3670 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3671 {
3672 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3673 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3674 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3675 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3676 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3677 }
3678
3679 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3680 AssertRCReturn(rc, rc);
3681
3682 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3683 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3684 }
3685 return rc;
3686}
3687
3688
3689/**
3690 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3691 *
3692 * @returns VBox status code.
3693 * @param pVCpu The cross context virtual CPU structure.
3694 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3695 * out-of-sync. Make sure to update the required fields
3696 * before using them.
3697 *
3698 * @remarks No-long-jump zone!!!
3699 */
3700DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3701{
3702 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3703 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3704 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3705 AssertRCReturn(rc, rc);
3706 return rc;
3707}
3708
3709
3710/**
3711 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3712 * CR0 is partially shared with the host and we have to consider the FPU bits.
3713 *
3714 * @returns VBox status code.
3715 * @param pVCpu The cross context virtual CPU structure.
3716 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3717 * out-of-sync. Make sure to update the required fields
3718 * before using them.
3719 *
3720 * @remarks No-long-jump zone!!!
3721 */
3722static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3723{
3724 /*
3725 * Guest CR0.
3726 * Guest FPU.
3727 */
3728 int rc = VINF_SUCCESS;
3729 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3730 {
3731 Assert(!(pMixedCtx->cr0 >> 32));
3732 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3733 PVM pVM = pVCpu->CTX_SUFF(pVM);
3734
3735 /* The guest's view (read access) of its CR0 is unblemished. */
3736 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3737 AssertRCReturn(rc, rc);
3738 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3739
3740 /* Setup VT-x's view of the guest CR0. */
3741 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3742 if (pVM->hm.s.fNestedPaging)
3743 {
3744 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3745 {
3746 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3747 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3748 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3749 }
3750 else
3751 {
3752 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3753 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3754 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3755 }
3756
3757 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3758 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3759 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3760
3761 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3762 AssertRCReturn(rc, rc);
3763 }
3764 else
3765 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3766
3767 /*
3768 * Guest FPU bits.
3769 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3770 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3771 */
3772 u32GuestCR0 |= X86_CR0_NE;
3773 bool fInterceptNM = false;
3774 if (CPUMIsGuestFPUStateActive(pVCpu))
3775 {
3776 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3777 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3778 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3779 }
3780 else
3781 {
3782 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3783 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3784 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3785 }
3786
3787 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3788 bool fInterceptMF = false;
3789 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3790 fInterceptMF = true;
3791
3792 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3793 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3794 {
3795 Assert(PDMVmmDevHeapIsEnabled(pVM));
3796 Assert(pVM->hm.s.vmx.pRealModeTSS);
3797 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3798 fInterceptNM = true;
3799 fInterceptMF = true;
3800 }
3801 else
3802 {
3803 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3804 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3805 }
3806 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3807
3808 if (fInterceptNM)
3809 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3810 else
3811 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3812
3813 if (fInterceptMF)
3814 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3815 else
3816 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3817
3818 /* Additional intercepts for debugging, define these yourself explicitly. */
3819#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3820 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3821 | RT_BIT(X86_XCPT_BP)
3822 | RT_BIT(X86_XCPT_DE)
3823 | RT_BIT(X86_XCPT_NM)
3824 | RT_BIT(X86_XCPT_TS)
3825 | RT_BIT(X86_XCPT_UD)
3826 | RT_BIT(X86_XCPT_NP)
3827 | RT_BIT(X86_XCPT_SS)
3828 | RT_BIT(X86_XCPT_GP)
3829 | RT_BIT(X86_XCPT_PF)
3830 | RT_BIT(X86_XCPT_MF)
3831 ;
3832#elif defined(HMVMX_ALWAYS_TRAP_PF)
3833 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3834#endif
3835
3836 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3837
3838 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3839 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3840 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3841 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3842 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3843 else
3844 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3845
3846 u32GuestCR0 |= uSetCR0;
3847 u32GuestCR0 &= uZapCR0;
3848 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3849
3850 /* Write VT-x's view of the guest CR0 into the VMCS. */
3851 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3852 AssertRCReturn(rc, rc);
3853 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3854 uZapCR0));
3855
3856 /*
3857 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3858 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3859 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3860 */
3861 uint32_t u32CR0Mask = 0;
3862 u32CR0Mask = X86_CR0_PE
3863 | X86_CR0_NE
3864 | X86_CR0_WP
3865 | X86_CR0_PG
3866 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3867 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3868 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3869
3870 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3871 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3872 * and @bugref{6944}. */
3873#if 0
3874 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3875 u32CR0Mask &= ~X86_CR0_PE;
3876#endif
3877 if (pVM->hm.s.fNestedPaging)
3878 u32CR0Mask &= ~X86_CR0_WP;
3879
3880 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3881 if (fInterceptNM)
3882 {
3883 u32CR0Mask |= X86_CR0_TS
3884 | X86_CR0_MP;
3885 }
3886
3887 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3888 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3889 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3890 AssertRCReturn(rc, rc);
3891 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3892
3893 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3894 }
3895 return rc;
3896}
3897
3898
3899/**
3900 * Loads the guest control registers (CR3, CR4) into the guest-state area
3901 * in the VMCS.
3902 *
3903 * @returns VBox strict status code.
3904 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3905 * without unrestricted guest access and the VMMDev is not presently
3906 * mapped (e.g. EFI32).
3907 *
3908 * @param pVCpu The cross context virtual CPU structure.
3909 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3910 * out-of-sync. Make sure to update the required fields
3911 * before using them.
3912 *
3913 * @remarks No-long-jump zone!!!
3914 */
3915static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3916{
3917 int rc = VINF_SUCCESS;
3918 PVM pVM = pVCpu->CTX_SUFF(pVM);
3919
3920 /*
3921 * Guest CR2.
3922 * It's always loaded in the assembler code. Nothing to do here.
3923 */
3924
3925 /*
3926 * Guest CR3.
3927 */
3928 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3929 {
3930 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3931 if (pVM->hm.s.fNestedPaging)
3932 {
3933 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3934
3935 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3936 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3937 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3938 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3939
3940 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3941 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3942 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3943
3944 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3945 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3946 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3947 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3948 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3949 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3950 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3951
3952 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3953 AssertRCReturn(rc, rc);
3954 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3955
3956 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3957 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3958 {
3959 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3960 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3961 {
3962 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3963 AssertRCReturn(rc, rc);
3964 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3965 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3966 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3967 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3968 AssertRCReturn(rc, rc);
3969 }
3970
3971 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3972 have Unrestricted Execution to handle the guest when it's not using paging. */
3973 GCPhysGuestCR3 = pMixedCtx->cr3;
3974 }
3975 else
3976 {
3977 /*
3978 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3979 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3980 * EPT takes care of translating it to host-physical addresses.
3981 */
3982 RTGCPHYS GCPhys;
3983 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3984
3985 /* We obtain it here every time as the guest could have relocated this PCI region. */
3986 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3987 if (RT_SUCCESS(rc))
3988 { /* likely */ }
3989 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3990 {
3991 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3992 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3993 }
3994 else
3995 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3996
3997 GCPhysGuestCR3 = GCPhys;
3998 }
3999
4000 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4001 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4002 }
4003 else
4004 {
4005 /* Non-nested paging case, just use the hypervisor's CR3. */
4006 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4007
4008 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4009 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4010 }
4011 AssertRCReturn(rc, rc);
4012
4013 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4014 }
4015
4016 /*
4017 * Guest CR4.
4018 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4019 */
4020 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4021 {
4022 Assert(!(pMixedCtx->cr4 >> 32));
4023 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4024
4025 /* The guest's view of its CR4 is unblemished. */
4026 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4027 AssertRCReturn(rc, rc);
4028 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4029
4030 /* Setup VT-x's view of the guest CR4. */
4031 /*
4032 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4033 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4034 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4035 */
4036 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4037 {
4038 Assert(pVM->hm.s.vmx.pRealModeTSS);
4039 Assert(PDMVmmDevHeapIsEnabled(pVM));
4040 u32GuestCR4 &= ~X86_CR4_VME;
4041 }
4042
4043 if (pVM->hm.s.fNestedPaging)
4044 {
4045 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4046 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4047 {
4048 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4049 u32GuestCR4 |= X86_CR4_PSE;
4050 /* Our identity mapping is a 32-bit page directory. */
4051 u32GuestCR4 &= ~X86_CR4_PAE;
4052 }
4053 /* else use guest CR4.*/
4054 }
4055 else
4056 {
4057 /*
4058 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4059 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4060 */
4061 switch (pVCpu->hm.s.enmShadowMode)
4062 {
4063 case PGMMODE_REAL: /* Real-mode. */
4064 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4065 case PGMMODE_32_BIT: /* 32-bit paging. */
4066 {
4067 u32GuestCR4 &= ~X86_CR4_PAE;
4068 break;
4069 }
4070
4071 case PGMMODE_PAE: /* PAE paging. */
4072 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4073 {
4074 u32GuestCR4 |= X86_CR4_PAE;
4075 break;
4076 }
4077
4078 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4079 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4080#ifdef VBOX_ENABLE_64_BITS_GUESTS
4081 break;
4082#endif
4083 default:
4084 AssertFailed();
4085 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4086 }
4087 }
4088
4089 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4090 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4091 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4092 u32GuestCR4 |= uSetCR4;
4093 u32GuestCR4 &= uZapCR4;
4094
4095 /* Write VT-x's view of the guest CR4 into the VMCS. */
4096 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4097 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4098 AssertRCReturn(rc, rc);
4099
4100 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4101 uint32_t u32CR4Mask = X86_CR4_VME
4102 | X86_CR4_PAE
4103 | X86_CR4_PGE
4104 | X86_CR4_PSE
4105 | X86_CR4_VMXE;
4106 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4107 u32CR4Mask |= X86_CR4_OSXSAVE;
4108 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4109 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4110 AssertRCReturn(rc, rc);
4111
4112 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4113 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4114
4115 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4116 }
4117 return rc;
4118}
4119
4120
4121/**
4122 * Loads the guest debug registers into the guest-state area in the VMCS.
4123 *
4124 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4125 *
4126 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4127 *
4128 * @returns VBox status code.
4129 * @param pVCpu The cross context virtual CPU structure.
4130 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4131 * out-of-sync. Make sure to update the required fields
4132 * before using them.
4133 *
4134 * @remarks No-long-jump zone!!!
4135 */
4136static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4137{
4138 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4139 return VINF_SUCCESS;
4140
4141#ifdef VBOX_STRICT
4142 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4143 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4144 {
4145 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4146 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4147 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4148 }
4149#endif
4150
4151 int rc;
4152 PVM pVM = pVCpu->CTX_SUFF(pVM);
4153 bool fSteppingDB = false;
4154 bool fInterceptMovDRx = false;
4155 if (pVCpu->hm.s.fSingleInstruction)
4156 {
4157 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4158 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4159 {
4160 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4161 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4162 AssertRCReturn(rc, rc);
4163 Assert(fSteppingDB == false);
4164 }
4165 else
4166 {
4167 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4168 pVCpu->hm.s.fClearTrapFlag = true;
4169 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4170 fSteppingDB = true;
4171 }
4172 }
4173
4174 if ( fSteppingDB
4175 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4176 {
4177 /*
4178 * Use the combined guest and host DRx values found in the hypervisor
4179 * register set because the debugger has breakpoints active or someone
4180 * is single stepping on the host side without a monitor trap flag.
4181 *
4182 * Note! DBGF expects a clean DR6 state before executing guest code.
4183 */
4184#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4185 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4186 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4187 {
4188 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4189 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4190 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4191 }
4192 else
4193#endif
4194 if (!CPUMIsHyperDebugStateActive(pVCpu))
4195 {
4196 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4197 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4198 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4199 }
4200
4201 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4202 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4203 AssertRCReturn(rc, rc);
4204
4205 pVCpu->hm.s.fUsingHyperDR7 = true;
4206 fInterceptMovDRx = true;
4207 }
4208 else
4209 {
4210 /*
4211 * If the guest has enabled debug registers, we need to load them prior to
4212 * executing guest code so they'll trigger at the right time.
4213 */
4214 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4215 {
4216#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4217 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4218 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4219 {
4220 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4221 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4222 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4223 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4224 }
4225 else
4226#endif
4227 if (!CPUMIsGuestDebugStateActive(pVCpu))
4228 {
4229 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4230 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4231 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4232 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4233 }
4234 Assert(!fInterceptMovDRx);
4235 }
4236 /*
4237 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4238 * must intercept #DB in order to maintain a correct DR6 guest value, and
4239 * because we need to intercept it to prevent nested #DBs from hanging the
4240 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4241 */
4242#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4243 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4244 && !CPUMIsGuestDebugStateActive(pVCpu))
4245#else
4246 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4247#endif
4248 {
4249 fInterceptMovDRx = true;
4250 }
4251
4252 /* Update guest DR7. */
4253 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4254 AssertRCReturn(rc, rc);
4255
4256 pVCpu->hm.s.fUsingHyperDR7 = false;
4257 }
4258
4259 /*
4260 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4261 */
4262 if (fInterceptMovDRx)
4263 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4264 else
4265 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4266 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4267 AssertRCReturn(rc, rc);
4268
4269 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4270 return VINF_SUCCESS;
4271}
4272
4273
4274#ifdef VBOX_STRICT
4275/**
4276 * Strict function to validate segment registers.
4277 *
4278 * @remarks ASSUMES CR0 is up to date.
4279 */
4280static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4281{
4282 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4283 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4284 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4285 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4286 && ( !CPUMIsGuestInRealModeEx(pCtx)
4287 && !CPUMIsGuestInV86ModeEx(pCtx)))
4288 {
4289 /* Protected mode checks */
4290 /* CS */
4291 Assert(pCtx->cs.Attr.n.u1Present);
4292 Assert(!(pCtx->cs.Attr.u & 0xf00));
4293 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4294 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4295 || !(pCtx->cs.Attr.n.u1Granularity));
4296 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4297 || (pCtx->cs.Attr.n.u1Granularity));
4298 /* CS cannot be loaded with NULL in protected mode. */
4299 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4300 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4301 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4302 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4303 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4304 else
4305 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4306 /* SS */
4307 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4308 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4309 if ( !(pCtx->cr0 & X86_CR0_PE)
4310 || pCtx->cs.Attr.n.u4Type == 3)
4311 {
4312 Assert(!pCtx->ss.Attr.n.u2Dpl);
4313 }
4314 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4315 {
4316 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4317 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4318 Assert(pCtx->ss.Attr.n.u1Present);
4319 Assert(!(pCtx->ss.Attr.u & 0xf00));
4320 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4321 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4322 || !(pCtx->ss.Attr.n.u1Granularity));
4323 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4324 || (pCtx->ss.Attr.n.u1Granularity));
4325 }
4326 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4327 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4328 {
4329 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4330 Assert(pCtx->ds.Attr.n.u1Present);
4331 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4332 Assert(!(pCtx->ds.Attr.u & 0xf00));
4333 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4334 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4335 || !(pCtx->ds.Attr.n.u1Granularity));
4336 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4337 || (pCtx->ds.Attr.n.u1Granularity));
4338 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4339 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4340 }
4341 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4342 {
4343 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4344 Assert(pCtx->es.Attr.n.u1Present);
4345 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4346 Assert(!(pCtx->es.Attr.u & 0xf00));
4347 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4348 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4349 || !(pCtx->es.Attr.n.u1Granularity));
4350 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4351 || (pCtx->es.Attr.n.u1Granularity));
4352 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4353 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4354 }
4355 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4356 {
4357 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4358 Assert(pCtx->fs.Attr.n.u1Present);
4359 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4360 Assert(!(pCtx->fs.Attr.u & 0xf00));
4361 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4362 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4363 || !(pCtx->fs.Attr.n.u1Granularity));
4364 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4365 || (pCtx->fs.Attr.n.u1Granularity));
4366 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4367 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4368 }
4369 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4370 {
4371 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4372 Assert(pCtx->gs.Attr.n.u1Present);
4373 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4374 Assert(!(pCtx->gs.Attr.u & 0xf00));
4375 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4376 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4377 || !(pCtx->gs.Attr.n.u1Granularity));
4378 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4379 || (pCtx->gs.Attr.n.u1Granularity));
4380 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4381 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4382 }
4383 /* 64-bit capable CPUs. */
4384# if HC_ARCH_BITS == 64
4385 Assert(!(pCtx->cs.u64Base >> 32));
4386 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4387 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4388 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4389# endif
4390 }
4391 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4392 || ( CPUMIsGuestInRealModeEx(pCtx)
4393 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4394 {
4395 /* Real and v86 mode checks. */
4396 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4397 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4398 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4399 {
4400 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4401 }
4402 else
4403 {
4404 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4405 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4406 }
4407
4408 /* CS */
4409 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4410 Assert(pCtx->cs.u32Limit == 0xffff);
4411 Assert(u32CSAttr == 0xf3);
4412 /* SS */
4413 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4414 Assert(pCtx->ss.u32Limit == 0xffff);
4415 Assert(u32SSAttr == 0xf3);
4416 /* DS */
4417 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4418 Assert(pCtx->ds.u32Limit == 0xffff);
4419 Assert(u32DSAttr == 0xf3);
4420 /* ES */
4421 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4422 Assert(pCtx->es.u32Limit == 0xffff);
4423 Assert(u32ESAttr == 0xf3);
4424 /* FS */
4425 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4426 Assert(pCtx->fs.u32Limit == 0xffff);
4427 Assert(u32FSAttr == 0xf3);
4428 /* GS */
4429 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4430 Assert(pCtx->gs.u32Limit == 0xffff);
4431 Assert(u32GSAttr == 0xf3);
4432 /* 64-bit capable CPUs. */
4433# if HC_ARCH_BITS == 64
4434 Assert(!(pCtx->cs.u64Base >> 32));
4435 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4436 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4437 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4438# endif
4439 }
4440}
4441#endif /* VBOX_STRICT */
4442
4443
4444/**
4445 * Writes a guest segment register into the guest-state area in the VMCS.
4446 *
4447 * @returns VBox status code.
4448 * @param pVCpu The cross context virtual CPU structure.
4449 * @param idxSel Index of the selector in the VMCS.
4450 * @param idxLimit Index of the segment limit in the VMCS.
4451 * @param idxBase Index of the segment base in the VMCS.
4452 * @param idxAccess Index of the access rights of the segment in the VMCS.
4453 * @param pSelReg Pointer to the segment selector.
4454 *
4455 * @remarks No-long-jump zone!!!
4456 */
4457static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4458 uint32_t idxAccess, PCPUMSELREG pSelReg)
4459{
4460 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4461 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4462 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4463 AssertRCReturn(rc, rc);
4464
4465 uint32_t u32Access = pSelReg->Attr.u;
4466 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4467 {
4468 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4469 u32Access = 0xf3;
4470 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4471 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4472 }
4473 else
4474 {
4475 /*
4476 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4477 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4478 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4479 * loaded in protected-mode have their attribute as 0.
4480 */
4481 if (!u32Access)
4482 u32Access = X86DESCATTR_UNUSABLE;
4483 }
4484
4485 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4486 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4487 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4488
4489 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4490 AssertRCReturn(rc, rc);
4491 return rc;
4492}
4493
4494
4495/**
4496 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4497 * into the guest-state area in the VMCS.
4498 *
4499 * @returns VBox status code.
4500 * @param pVCpu The cross context virtual CPU structure.
4501 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4502 * out-of-sync. Make sure to update the required fields
4503 * before using them.
4504 *
4505 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4506 * @remarks No-long-jump zone!!!
4507 */
4508static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4509{
4510 int rc = VERR_INTERNAL_ERROR_5;
4511 PVM pVM = pVCpu->CTX_SUFF(pVM);
4512
4513 /*
4514 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4515 */
4516 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4517 {
4518 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4519 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4520 {
4521 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4522 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4523 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4524 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4525 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4526 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4527 }
4528
4529#ifdef VBOX_WITH_REM
4530 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4531 {
4532 Assert(pVM->hm.s.vmx.pRealModeTSS);
4533 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4534 if ( pVCpu->hm.s.vmx.fWasInRealMode
4535 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4536 {
4537 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4538 in real-mode (e.g. OpenBSD 4.0) */
4539 REMFlushTBs(pVM);
4540 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4541 pVCpu->hm.s.vmx.fWasInRealMode = false;
4542 }
4543 }
4544#endif
4545 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4546 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4547 AssertRCReturn(rc, rc);
4548 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4549 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4550 AssertRCReturn(rc, rc);
4551 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4552 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4553 AssertRCReturn(rc, rc);
4554 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4555 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4556 AssertRCReturn(rc, rc);
4557 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4558 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4559 AssertRCReturn(rc, rc);
4560 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4561 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4562 AssertRCReturn(rc, rc);
4563
4564#ifdef VBOX_STRICT
4565 /* Validate. */
4566 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4567#endif
4568
4569 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4570 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4571 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4572 }
4573
4574 /*
4575 * Guest TR.
4576 */
4577 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4578 {
4579 /*
4580 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4581 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4582 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4583 */
4584 uint16_t u16Sel = 0;
4585 uint32_t u32Limit = 0;
4586 uint64_t u64Base = 0;
4587 uint32_t u32AccessRights = 0;
4588
4589 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4590 {
4591 u16Sel = pMixedCtx->tr.Sel;
4592 u32Limit = pMixedCtx->tr.u32Limit;
4593 u64Base = pMixedCtx->tr.u64Base;
4594 u32AccessRights = pMixedCtx->tr.Attr.u;
4595 }
4596 else
4597 {
4598 Assert(pVM->hm.s.vmx.pRealModeTSS);
4599 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4600
4601 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4602 RTGCPHYS GCPhys;
4603 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4604 AssertRCReturn(rc, rc);
4605
4606 X86DESCATTR DescAttr;
4607 DescAttr.u = 0;
4608 DescAttr.n.u1Present = 1;
4609 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4610
4611 u16Sel = 0;
4612 u32Limit = HM_VTX_TSS_SIZE;
4613 u64Base = GCPhys; /* in real-mode phys = virt. */
4614 u32AccessRights = DescAttr.u;
4615 }
4616
4617 /* Validate. */
4618 Assert(!(u16Sel & RT_BIT(2)));
4619 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4620 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4621 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4622 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4623 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4624 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4625 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4626 Assert( (u32Limit & 0xfff) == 0xfff
4627 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4628 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4629 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4630
4631 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4632 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4633 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4634 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4635 AssertRCReturn(rc, rc);
4636
4637 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4638 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4639 }
4640
4641 /*
4642 * Guest GDTR.
4643 */
4644 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4645 {
4646 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4647 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4648 AssertRCReturn(rc, rc);
4649
4650 /* Validate. */
4651 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4652
4653 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4654 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4655 }
4656
4657 /*
4658 * Guest LDTR.
4659 */
4660 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4661 {
4662 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4663 uint32_t u32Access = 0;
4664 if (!pMixedCtx->ldtr.Attr.u)
4665 u32Access = X86DESCATTR_UNUSABLE;
4666 else
4667 u32Access = pMixedCtx->ldtr.Attr.u;
4668
4669 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4670 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4671 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4672 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4673 AssertRCReturn(rc, rc);
4674
4675 /* Validate. */
4676 if (!(u32Access & X86DESCATTR_UNUSABLE))
4677 {
4678 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4679 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4680 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4681 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4682 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4683 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4684 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4685 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4686 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4687 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4688 }
4689
4690 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4691 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4692 }
4693
4694 /*
4695 * Guest IDTR.
4696 */
4697 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4698 {
4699 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4700 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4701 AssertRCReturn(rc, rc);
4702
4703 /* Validate. */
4704 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4705
4706 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4707 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4708 }
4709
4710 return VINF_SUCCESS;
4711}
4712
4713
4714/**
4715 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4716 * areas.
4717 *
4718 * These MSRs will automatically be loaded to the host CPU on every successful
4719 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4720 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4721 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4722 *
4723 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4724 *
4725 * @returns VBox status code.
4726 * @param pVCpu The cross context virtual CPU structure.
4727 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4728 * out-of-sync. Make sure to update the required fields
4729 * before using them.
4730 *
4731 * @remarks No-long-jump zone!!!
4732 */
4733static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4734{
4735 AssertPtr(pVCpu);
4736 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4737
4738 /*
4739 * MSRs that we use the auto-load/store MSR area in the VMCS.
4740 */
4741 PVM pVM = pVCpu->CTX_SUFF(pVM);
4742 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4743 {
4744 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4745#if HC_ARCH_BITS == 32
4746 if (pVM->hm.s.fAllow64BitGuests)
4747 {
4748 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4749 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4750 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4751 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4752 AssertRCReturn(rc, rc);
4753# ifdef LOG_ENABLED
4754 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4755 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4756 {
4757 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4758 pMsr->u64Value));
4759 }
4760# endif
4761 }
4762#endif
4763 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4764 }
4765
4766 /*
4767 * Guest Sysenter MSRs.
4768 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4769 * VM-exits on WRMSRs for these MSRs.
4770 */
4771 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4772 {
4773 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4774 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4775 }
4776
4777 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4778 {
4779 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4780 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4781 }
4782
4783 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4784 {
4785 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4786 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4787 }
4788
4789 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4790 {
4791 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4792 {
4793 /*
4794 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4795 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4796 */
4797 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4798 {
4799 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4800 AssertRCReturn(rc,rc);
4801 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4802 }
4803 else
4804 {
4805 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4806 NULL /* pfAddedAndUpdated */);
4807 AssertRCReturn(rc, rc);
4808
4809 /* We need to intercept reads too, see @bugref{7386#c16}. */
4810 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4811 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4812 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4813 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4814 }
4815 }
4816 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4817 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4818 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4819 }
4820
4821 return VINF_SUCCESS;
4822}
4823
4824
4825/**
4826 * Loads the guest activity state into the guest-state area in the VMCS.
4827 *
4828 * @returns VBox status code.
4829 * @param pVCpu The cross context virtual CPU structure.
4830 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4831 * out-of-sync. Make sure to update the required fields
4832 * before using them.
4833 *
4834 * @remarks No-long-jump zone!!!
4835 */
4836static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4837{
4838 NOREF(pMixedCtx);
4839 /** @todo See if we can make use of other states, e.g.
4840 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4841 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4842 {
4843 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4844 AssertRCReturn(rc, rc);
4845
4846 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4847 }
4848 return VINF_SUCCESS;
4849}
4850
4851
4852#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4853/**
4854 * Check if guest state allows safe use of 32-bit switcher again.
4855 *
4856 * Segment bases and protected mode structures must be 32-bit addressable
4857 * because the 32-bit switcher will ignore high dword when writing these VMCS
4858 * fields. See @bugref{8432} for details.
4859 *
4860 * @returns true if safe, false if must continue to use the 64-bit switcher.
4861 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4862 * out-of-sync. Make sure to update the required fields
4863 * before using them.
4864 *
4865 * @remarks No-long-jump zone!!!
4866 */
4867static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4868{
4869 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4870 return false;
4871 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4872 return false;
4873 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4874 return false;
4875 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4876 return false;
4877 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4878 return false;
4879 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4880 return false;
4881 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4882 return false;
4883 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4884 return false;
4885 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4886 return false;
4887 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4888 return false;
4889 /* All good, bases are 32-bit. */
4890 return true;
4891}
4892#endif
4893
4894
4895/**
4896 * Sets up the appropriate function to run guest code.
4897 *
4898 * @returns VBox status code.
4899 * @param pVCpu The cross context virtual CPU structure.
4900 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4901 * out-of-sync. Make sure to update the required fields
4902 * before using them.
4903 *
4904 * @remarks No-long-jump zone!!!
4905 */
4906static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4907{
4908 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4909 {
4910#ifndef VBOX_ENABLE_64_BITS_GUESTS
4911 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4912#endif
4913 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4914#if HC_ARCH_BITS == 32
4915 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4916 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4917 {
4918 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4919 {
4920 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4921 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4922 | HM_CHANGED_VMX_ENTRY_CTLS
4923 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4924 }
4925 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4926
4927 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4928 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4929 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4930 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4931 }
4932#else
4933 /* 64-bit host. */
4934 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4935#endif
4936 }
4937 else
4938 {
4939 /* Guest is not in long mode, use the 32-bit handler. */
4940#if HC_ARCH_BITS == 32
4941 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4942 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4943 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4944 {
4945 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4946 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4947 | HM_CHANGED_VMX_ENTRY_CTLS
4948 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4949 }
4950# ifdef VBOX_ENABLE_64_BITS_GUESTS
4951 /*
4952 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4953 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4954 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4955 * the much faster 32-bit switcher again.
4956 */
4957 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4958 {
4959 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4960 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4961 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4962 }
4963 else
4964 {
4965 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4966 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4967 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4968 {
4969 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4970 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4971 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4972 | HM_CHANGED_VMX_ENTRY_CTLS
4973 | HM_CHANGED_VMX_EXIT_CTLS
4974 | HM_CHANGED_HOST_CONTEXT);
4975 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4976 }
4977 }
4978# else
4979 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4980# endif
4981#else
4982 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4983#endif
4984 }
4985 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4986 return VINF_SUCCESS;
4987}
4988
4989
4990/**
4991 * Wrapper for running the guest code in VT-x.
4992 *
4993 * @returns VBox status code, no informational status codes.
4994 * @param pVM The cross context VM structure.
4995 * @param pVCpu The cross context virtual CPU structure.
4996 * @param pCtx Pointer to the guest-CPU context.
4997 *
4998 * @remarks No-long-jump zone!!!
4999 */
5000DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5001{
5002 /*
5003 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5004 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5005 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5006 */
5007 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5008 /** @todo Add stats for resume vs launch. */
5009#ifdef VBOX_WITH_KERNEL_USING_XMM
5010 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5011#else
5012 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5013#endif
5014 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5015 return rc;
5016}
5017
5018
5019/**
5020 * Reports world-switch error and dumps some useful debug info.
5021 *
5022 * @param pVM The cross context VM structure.
5023 * @param pVCpu The cross context virtual CPU structure.
5024 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5025 * @param pCtx Pointer to the guest-CPU context.
5026 * @param pVmxTransient Pointer to the VMX transient structure (only
5027 * exitReason updated).
5028 */
5029static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5030{
5031 Assert(pVM);
5032 Assert(pVCpu);
5033 Assert(pCtx);
5034 Assert(pVmxTransient);
5035 HMVMX_ASSERT_PREEMPT_SAFE();
5036
5037 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5038 switch (rcVMRun)
5039 {
5040 case VERR_VMX_INVALID_VMXON_PTR:
5041 AssertFailed();
5042 break;
5043 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5044 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5045 {
5046 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5047 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5048 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5049 AssertRC(rc);
5050
5051 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5052 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5053 Cannot do it here as we may have been long preempted. */
5054
5055#ifdef VBOX_STRICT
5056 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5057 pVmxTransient->uExitReason));
5058 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5059 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5060 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5061 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5062 else
5063 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5064 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5065 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5066
5067 /* VMX control bits. */
5068 uint32_t u32Val;
5069 uint64_t u64Val;
5070 RTHCUINTREG uHCReg;
5071 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5072 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5073 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5074 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5075 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5076 {
5077 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5078 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5079 }
5080 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5081 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5082 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5083 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5084 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5085 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5086 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5087 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5088 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5089 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5090 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5091 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5092 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5093 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5095 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5096 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5097 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5098 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5099 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5100 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5101 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5102 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5103 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5104 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5105 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5106 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5107 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5108 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5109 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5110 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5111 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5112 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5113 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5114 if (pVM->hm.s.fNestedPaging)
5115 {
5116 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5117 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5118 }
5119
5120 /* Guest bits. */
5121 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5122 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5123 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5124 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5125 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5126 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5127 if (pVM->hm.s.vmx.fVpid)
5128 {
5129 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5130 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5131 }
5132
5133 /* Host bits. */
5134 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5135 Log4(("Host CR0 %#RHr\n", uHCReg));
5136 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5137 Log4(("Host CR3 %#RHr\n", uHCReg));
5138 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5139 Log4(("Host CR4 %#RHr\n", uHCReg));
5140
5141 RTGDTR HostGdtr;
5142 PCX86DESCHC pDesc;
5143 ASMGetGDTR(&HostGdtr);
5144 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5145 Log4(("Host CS %#08x\n", u32Val));
5146 if (u32Val < HostGdtr.cbGdt)
5147 {
5148 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5149 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5150 }
5151
5152 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5153 Log4(("Host DS %#08x\n", u32Val));
5154 if (u32Val < HostGdtr.cbGdt)
5155 {
5156 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5157 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5158 }
5159
5160 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5161 Log4(("Host ES %#08x\n", u32Val));
5162 if (u32Val < HostGdtr.cbGdt)
5163 {
5164 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5165 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5166 }
5167
5168 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5169 Log4(("Host FS %#08x\n", u32Val));
5170 if (u32Val < HostGdtr.cbGdt)
5171 {
5172 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5173 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5174 }
5175
5176 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5177 Log4(("Host GS %#08x\n", u32Val));
5178 if (u32Val < HostGdtr.cbGdt)
5179 {
5180 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5181 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5182 }
5183
5184 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5185 Log4(("Host SS %#08x\n", u32Val));
5186 if (u32Val < HostGdtr.cbGdt)
5187 {
5188 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5189 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5190 }
5191
5192 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5193 Log4(("Host TR %#08x\n", u32Val));
5194 if (u32Val < HostGdtr.cbGdt)
5195 {
5196 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5197 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5198 }
5199
5200 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5201 Log4(("Host TR Base %#RHv\n", uHCReg));
5202 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5203 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5204 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5205 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5206 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5207 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5208 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5209 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5210 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5211 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5212 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5213 Log4(("Host RSP %#RHv\n", uHCReg));
5214 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5215 Log4(("Host RIP %#RHv\n", uHCReg));
5216# if HC_ARCH_BITS == 64
5217 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5218 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5219 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5220 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5221 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5222 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5223# endif
5224#endif /* VBOX_STRICT */
5225 break;
5226 }
5227
5228 default:
5229 /* Impossible */
5230 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5231 break;
5232 }
5233 NOREF(pVM); NOREF(pCtx);
5234}
5235
5236
5237#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5238#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5239# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5240#endif
5241#ifdef VBOX_STRICT
5242static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5243{
5244 switch (idxField)
5245 {
5246 case VMX_VMCS_GUEST_RIP:
5247 case VMX_VMCS_GUEST_RSP:
5248 case VMX_VMCS_GUEST_SYSENTER_EIP:
5249 case VMX_VMCS_GUEST_SYSENTER_ESP:
5250 case VMX_VMCS_GUEST_GDTR_BASE:
5251 case VMX_VMCS_GUEST_IDTR_BASE:
5252 case VMX_VMCS_GUEST_CS_BASE:
5253 case VMX_VMCS_GUEST_DS_BASE:
5254 case VMX_VMCS_GUEST_ES_BASE:
5255 case VMX_VMCS_GUEST_FS_BASE:
5256 case VMX_VMCS_GUEST_GS_BASE:
5257 case VMX_VMCS_GUEST_SS_BASE:
5258 case VMX_VMCS_GUEST_LDTR_BASE:
5259 case VMX_VMCS_GUEST_TR_BASE:
5260 case VMX_VMCS_GUEST_CR3:
5261 return true;
5262 }
5263 return false;
5264}
5265
5266static bool hmR0VmxIsValidReadField(uint32_t idxField)
5267{
5268 switch (idxField)
5269 {
5270 /* Read-only fields. */
5271 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5272 return true;
5273 }
5274 /* Remaining readable fields should also be writable. */
5275 return hmR0VmxIsValidWriteField(idxField);
5276}
5277#endif /* VBOX_STRICT */
5278
5279
5280/**
5281 * Executes the specified handler in 64-bit mode.
5282 *
5283 * @returns VBox status code (no informational status codes).
5284 * @param pVM The cross context VM structure.
5285 * @param pVCpu The cross context virtual CPU structure.
5286 * @param pCtx Pointer to the guest CPU context.
5287 * @param enmOp The operation to perform.
5288 * @param cParams Number of parameters.
5289 * @param paParam Array of 32-bit parameters.
5290 */
5291VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5292 uint32_t cParams, uint32_t *paParam)
5293{
5294 NOREF(pCtx);
5295
5296 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5297 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5298 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5299 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5300
5301#ifdef VBOX_STRICT
5302 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5303 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5304
5305 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5306 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5307#endif
5308
5309 /* Disable interrupts. */
5310 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5311
5312#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5313 RTCPUID idHostCpu = RTMpCpuId();
5314 CPUMR0SetLApic(pVCpu, idHostCpu);
5315#endif
5316
5317 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5318 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5319
5320 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5321 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5322 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5323
5324 /* Leave VMX Root Mode. */
5325 VMXDisable();
5326
5327 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5328
5329 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5330 CPUMSetHyperEIP(pVCpu, enmOp);
5331 for (int i = (int)cParams - 1; i >= 0; i--)
5332 CPUMPushHyper(pVCpu, paParam[i]);
5333
5334 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5335
5336 /* Call the switcher. */
5337 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5338 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5339
5340 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5341 /* Make sure the VMX instructions don't cause #UD faults. */
5342 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5343
5344 /* Re-enter VMX Root Mode */
5345 int rc2 = VMXEnable(HCPhysCpuPage);
5346 if (RT_FAILURE(rc2))
5347 {
5348 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5349 ASMSetFlags(fOldEFlags);
5350 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5351 return rc2;
5352 }
5353
5354 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5355 AssertRC(rc2);
5356 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5357 Assert(!(ASMGetFlags() & X86_EFL_IF));
5358 ASMSetFlags(fOldEFlags);
5359 return rc;
5360}
5361
5362
5363/**
5364 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5365 * supporting 64-bit guests.
5366 *
5367 * @returns VBox status code.
5368 * @param fResume Whether to VMLAUNCH or VMRESUME.
5369 * @param pCtx Pointer to the guest-CPU context.
5370 * @param pCache Pointer to the VMCS cache.
5371 * @param pVM The cross context VM structure.
5372 * @param pVCpu The cross context virtual CPU structure.
5373 */
5374DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5375{
5376 NOREF(fResume);
5377
5378 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5379 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5380
5381#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5382 pCache->uPos = 1;
5383 pCache->interPD = PGMGetInterPaeCR3(pVM);
5384 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5385#endif
5386
5387#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5388 pCache->TestIn.HCPhysCpuPage = 0;
5389 pCache->TestIn.HCPhysVmcs = 0;
5390 pCache->TestIn.pCache = 0;
5391 pCache->TestOut.HCPhysVmcs = 0;
5392 pCache->TestOut.pCache = 0;
5393 pCache->TestOut.pCtx = 0;
5394 pCache->TestOut.eflags = 0;
5395#else
5396 NOREF(pCache);
5397#endif
5398
5399 uint32_t aParam[10];
5400 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5401 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5402 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5403 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5404 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5405 aParam[5] = 0;
5406 aParam[6] = VM_RC_ADDR(pVM, pVM);
5407 aParam[7] = 0;
5408 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5409 aParam[9] = 0;
5410
5411#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5412 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5413 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5414#endif
5415 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5416
5417#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5418 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5419 Assert(pCtx->dr[4] == 10);
5420 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5421#endif
5422
5423#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5424 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5425 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5426 pVCpu->hm.s.vmx.HCPhysVmcs));
5427 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5428 pCache->TestOut.HCPhysVmcs));
5429 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5430 pCache->TestOut.pCache));
5431 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5432 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5433 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5434 pCache->TestOut.pCtx));
5435 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5436#endif
5437 return rc;
5438}
5439
5440
5441/**
5442 * Initialize the VMCS-Read cache.
5443 *
5444 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5445 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5446 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5447 * (those that have a 32-bit FULL & HIGH part).
5448 *
5449 * @returns VBox status code.
5450 * @param pVM The cross context VM structure.
5451 * @param pVCpu The cross context virtual CPU structure.
5452 */
5453static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5454{
5455#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5456{ \
5457 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5458 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5459 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5460 ++cReadFields; \
5461}
5462
5463 AssertPtr(pVM);
5464 AssertPtr(pVCpu);
5465 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5466 uint32_t cReadFields = 0;
5467
5468 /*
5469 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5470 * and serve to indicate exceptions to the rules.
5471 */
5472
5473 /* Guest-natural selector base fields. */
5474#if 0
5475 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5476 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5478#endif
5479 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5480 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5481 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5482 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5485 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5486 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5487 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5488 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5491#if 0
5492 /* Unused natural width guest-state fields. */
5493 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5494 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5495#endif
5496 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5497 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5498
5499 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5500#if 0
5501 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5502 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5503 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5504 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5505 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5506 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5507 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5508 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5509 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5510#endif
5511
5512 /* Natural width guest-state fields. */
5513 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5514#if 0
5515 /* Currently unused field. */
5516 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5517#endif
5518
5519 if (pVM->hm.s.fNestedPaging)
5520 {
5521 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5522 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5523 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5524 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5525 }
5526 else
5527 {
5528 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5529 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5530 }
5531
5532#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5533 return VINF_SUCCESS;
5534}
5535
5536
5537/**
5538 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5539 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5540 * darwin, running 64-bit guests).
5541 *
5542 * @returns VBox status code.
5543 * @param pVCpu The cross context virtual CPU structure.
5544 * @param idxField The VMCS field encoding.
5545 * @param u64Val 16, 32 or 64-bit value.
5546 */
5547VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5548{
5549 int rc;
5550 switch (idxField)
5551 {
5552 /*
5553 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5554 */
5555 /* 64-bit Control fields. */
5556 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5557 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5558 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5559 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5560 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5561 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5562 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5563 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5564 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5565 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5566 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5567 case VMX_VMCS64_CTRL_EPTP_FULL:
5568 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5569 /* 64-bit Guest-state fields. */
5570 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5571 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5572 case VMX_VMCS64_GUEST_PAT_FULL:
5573 case VMX_VMCS64_GUEST_EFER_FULL:
5574 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5575 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5576 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5577 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5578 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5579 /* 64-bit Host-state fields. */
5580 case VMX_VMCS64_HOST_PAT_FULL:
5581 case VMX_VMCS64_HOST_EFER_FULL:
5582 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5583 {
5584 rc = VMXWriteVmcs32(idxField, u64Val);
5585 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5586 break;
5587 }
5588
5589 /*
5590 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5591 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5592 */
5593 /* Natural-width Guest-state fields. */
5594 case VMX_VMCS_GUEST_CR3:
5595 case VMX_VMCS_GUEST_ES_BASE:
5596 case VMX_VMCS_GUEST_CS_BASE:
5597 case VMX_VMCS_GUEST_SS_BASE:
5598 case VMX_VMCS_GUEST_DS_BASE:
5599 case VMX_VMCS_GUEST_FS_BASE:
5600 case VMX_VMCS_GUEST_GS_BASE:
5601 case VMX_VMCS_GUEST_LDTR_BASE:
5602 case VMX_VMCS_GUEST_TR_BASE:
5603 case VMX_VMCS_GUEST_GDTR_BASE:
5604 case VMX_VMCS_GUEST_IDTR_BASE:
5605 case VMX_VMCS_GUEST_RSP:
5606 case VMX_VMCS_GUEST_RIP:
5607 case VMX_VMCS_GUEST_SYSENTER_ESP:
5608 case VMX_VMCS_GUEST_SYSENTER_EIP:
5609 {
5610 if (!(u64Val >> 32))
5611 {
5612 /* If this field is 64-bit, VT-x will zero out the top bits. */
5613 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5614 }
5615 else
5616 {
5617 /* Assert that only the 32->64 switcher case should ever come here. */
5618 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5619 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5620 }
5621 break;
5622 }
5623
5624 default:
5625 {
5626 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5627 rc = VERR_INVALID_PARAMETER;
5628 break;
5629 }
5630 }
5631 AssertRCReturn(rc, rc);
5632 return rc;
5633}
5634
5635
5636/**
5637 * Queue up a VMWRITE by using the VMCS write cache.
5638 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5639 *
5640 * @param pVCpu The cross context virtual CPU structure.
5641 * @param idxField The VMCS field encoding.
5642 * @param u64Val 16, 32 or 64-bit value.
5643 */
5644VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5645{
5646 AssertPtr(pVCpu);
5647 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5648
5649 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5650 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5651
5652 /* Make sure there are no duplicates. */
5653 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5654 {
5655 if (pCache->Write.aField[i] == idxField)
5656 {
5657 pCache->Write.aFieldVal[i] = u64Val;
5658 return VINF_SUCCESS;
5659 }
5660 }
5661
5662 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5663 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5664 pCache->Write.cValidEntries++;
5665 return VINF_SUCCESS;
5666}
5667#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5668
5669
5670/**
5671 * Sets up the usage of TSC-offsetting and updates the VMCS.
5672 *
5673 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5674 * VMX preemption timer.
5675 *
5676 * @returns VBox status code.
5677 * @param pVM The cross context VM structure.
5678 * @param pVCpu The cross context virtual CPU structure.
5679 *
5680 * @remarks No-long-jump zone!!!
5681 */
5682static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5683{
5684 int rc;
5685 bool fOffsettedTsc;
5686 bool fParavirtTsc;
5687 if (pVM->hm.s.vmx.fUsePreemptTimer)
5688 {
5689 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5690 &fOffsettedTsc, &fParavirtTsc);
5691
5692 /* Make sure the returned values have sane upper and lower boundaries. */
5693 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5694 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5695 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5696 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5697
5698 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5699 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5700 }
5701 else
5702 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5703
5704 /** @todo later optimize this to be done elsewhere and not before every
5705 * VM-entry. */
5706 if (fParavirtTsc)
5707 {
5708 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5709 information before every VM-entry, hence disable it for performance sake. */
5710#if 0
5711 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5712 AssertRC(rc);
5713#endif
5714 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5715 }
5716
5717 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5718 {
5719 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5720 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5721
5722 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5723 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5724 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5725 }
5726 else
5727 {
5728 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5729 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5730 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5731 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5732 }
5733}
5734
5735
5736/**
5737 * Determines if an exception is a contributory exception.
5738 *
5739 * Contributory exceptions are ones which can cause double-faults unless the
5740 * original exception was a benign exception. Page-fault is intentionally not
5741 * included here as it's a conditional contributory exception.
5742 *
5743 * @returns true if the exception is contributory, false otherwise.
5744 * @param uVector The exception vector.
5745 */
5746DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5747{
5748 switch (uVector)
5749 {
5750 case X86_XCPT_GP:
5751 case X86_XCPT_SS:
5752 case X86_XCPT_NP:
5753 case X86_XCPT_TS:
5754 case X86_XCPT_DE:
5755 return true;
5756 default:
5757 break;
5758 }
5759 return false;
5760}
5761
5762
5763/**
5764 * Sets an event as a pending event to be injected into the guest.
5765 *
5766 * @param pVCpu The cross context virtual CPU structure.
5767 * @param u32IntInfo The VM-entry interruption-information field.
5768 * @param cbInstr The VM-entry instruction length in bytes (for software
5769 * interrupts, exceptions and privileged software
5770 * exceptions).
5771 * @param u32ErrCode The VM-entry exception error code.
5772 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5773 * page-fault.
5774 *
5775 * @remarks Statistics counter assumes this is a guest event being injected or
5776 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5777 * always incremented.
5778 */
5779DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5780 RTGCUINTPTR GCPtrFaultAddress)
5781{
5782 Assert(!pVCpu->hm.s.Event.fPending);
5783 pVCpu->hm.s.Event.fPending = true;
5784 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5785 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5786 pVCpu->hm.s.Event.cbInstr = cbInstr;
5787 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5788}
5789
5790
5791/**
5792 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5793 *
5794 * @param pVCpu The cross context virtual CPU structure.
5795 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5796 * out-of-sync. Make sure to update the required fields
5797 * before using them.
5798 */
5799DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5800{
5801 NOREF(pMixedCtx);
5802 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5803 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5804 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5805 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5806}
5807
5808
5809/**
5810 * Handle a condition that occurred while delivering an event through the guest
5811 * IDT.
5812 *
5813 * @returns Strict VBox status code (i.e. informational status codes too).
5814 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5815 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5816 * to continue execution of the guest which will delivery the \#DF.
5817 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5818 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5819 *
5820 * @param pVCpu The cross context virtual CPU structure.
5821 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5822 * out-of-sync. Make sure to update the required fields
5823 * before using them.
5824 * @param pVmxTransient Pointer to the VMX transient structure.
5825 *
5826 * @remarks No-long-jump zone!!!
5827 */
5828static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5829{
5830 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5831
5832 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5833 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5834
5835 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5836 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5837 {
5838 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5839 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5840
5841 typedef enum
5842 {
5843 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5844 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5845 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5846 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5847 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5848 } VMXREFLECTXCPT;
5849
5850 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5851 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5852 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5853 {
5854 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5855 {
5856 enmReflect = VMXREFLECTXCPT_XCPT;
5857#ifdef VBOX_STRICT
5858 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5859 && uExitVector == X86_XCPT_PF)
5860 {
5861 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5862 }
5863#endif
5864 if ( uExitVector == X86_XCPT_PF
5865 && uIdtVector == X86_XCPT_PF)
5866 {
5867 pVmxTransient->fVectoringDoublePF = true;
5868 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5869 }
5870 else if ( uExitVector == X86_XCPT_AC
5871 && uIdtVector == X86_XCPT_AC)
5872 {
5873 enmReflect = VMXREFLECTXCPT_HANG;
5874 Log4(("IDT: Nested #AC - Bad guest\n"));
5875 }
5876 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5877 && hmR0VmxIsContributoryXcpt(uExitVector)
5878 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5879 || uIdtVector == X86_XCPT_PF))
5880 {
5881 enmReflect = VMXREFLECTXCPT_DF;
5882 }
5883 else if (uIdtVector == X86_XCPT_DF)
5884 enmReflect = VMXREFLECTXCPT_TF;
5885 }
5886 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5887 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5888 {
5889 /*
5890 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5891 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5892 */
5893 enmReflect = VMXREFLECTXCPT_XCPT;
5894
5895 if (uExitVector == X86_XCPT_PF)
5896 {
5897 pVmxTransient->fVectoringPF = true;
5898 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5899 }
5900 }
5901 }
5902 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5903 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5904 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5905 {
5906 /*
5907 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5908 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5909 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5910 */
5911 enmReflect = VMXREFLECTXCPT_XCPT;
5912 }
5913
5914 /*
5915 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5916 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5917 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5918 *
5919 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5920 */
5921 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5922 && enmReflect == VMXREFLECTXCPT_XCPT
5923 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5924 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5925 {
5926 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5927 }
5928
5929 switch (enmReflect)
5930 {
5931 case VMXREFLECTXCPT_XCPT:
5932 {
5933 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5934 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5935 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5936
5937 uint32_t u32ErrCode = 0;
5938 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5939 {
5940 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5941 AssertRCReturn(rc2, rc2);
5942 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5943 }
5944
5945 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5946 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5947 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5948 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5949 rcStrict = VINF_SUCCESS;
5950 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5951 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5952
5953 break;
5954 }
5955
5956 case VMXREFLECTXCPT_DF:
5957 {
5958 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5959 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5960 rcStrict = VINF_HM_DOUBLE_FAULT;
5961 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5962 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5963
5964 break;
5965 }
5966
5967 case VMXREFLECTXCPT_TF:
5968 {
5969 rcStrict = VINF_EM_RESET;
5970 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5971 uExitVector));
5972 break;
5973 }
5974
5975 case VMXREFLECTXCPT_HANG:
5976 {
5977 rcStrict = VERR_EM_GUEST_CPU_HANG;
5978 break;
5979 }
5980
5981 default:
5982 Assert(rcStrict == VINF_SUCCESS);
5983 break;
5984 }
5985 }
5986 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5987 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5988 && uExitVector != X86_XCPT_DF
5989 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5990 {
5991 /*
5992 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5993 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5994 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5995 */
5996 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5997 {
5998 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5999 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6000 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6001 }
6002 }
6003
6004 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6005 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6006 return rcStrict;
6007}
6008
6009
6010/**
6011 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6012 *
6013 * @returns VBox status code.
6014 * @param pVCpu The cross context virtual CPU structure.
6015 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6016 * out-of-sync. Make sure to update the required fields
6017 * before using them.
6018 *
6019 * @remarks No-long-jump zone!!!
6020 */
6021static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6022{
6023 NOREF(pMixedCtx);
6024
6025 /*
6026 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6027 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6028 */
6029 VMMRZCallRing3Disable(pVCpu);
6030 HM_DISABLE_PREEMPT();
6031
6032 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6033 {
6034 uint32_t uVal = 0;
6035 uint32_t uShadow = 0;
6036 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6037 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6038 AssertRCReturn(rc, rc);
6039
6040 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6041 CPUMSetGuestCR0(pVCpu, uVal);
6042 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6043 }
6044
6045 HM_RESTORE_PREEMPT();
6046 VMMRZCallRing3Enable(pVCpu);
6047 return VINF_SUCCESS;
6048}
6049
6050
6051/**
6052 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6053 *
6054 * @returns VBox status code.
6055 * @param pVCpu The cross context virtual CPU structure.
6056 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6057 * out-of-sync. Make sure to update the required fields
6058 * before using them.
6059 *
6060 * @remarks No-long-jump zone!!!
6061 */
6062static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6063{
6064 NOREF(pMixedCtx);
6065
6066 int rc = VINF_SUCCESS;
6067 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6068 {
6069 uint32_t uVal = 0;
6070 uint32_t uShadow = 0;
6071 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6072 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6073 AssertRCReturn(rc, rc);
6074
6075 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6076 CPUMSetGuestCR4(pVCpu, uVal);
6077 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6078 }
6079 return rc;
6080}
6081
6082
6083/**
6084 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6085 *
6086 * @returns VBox status code.
6087 * @param pVCpu The cross context virtual CPU structure.
6088 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6089 * out-of-sync. Make sure to update the required fields
6090 * before using them.
6091 *
6092 * @remarks No-long-jump zone!!!
6093 */
6094static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6095{
6096 int rc = VINF_SUCCESS;
6097 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6098 {
6099 uint64_t u64Val = 0;
6100 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6101 AssertRCReturn(rc, rc);
6102
6103 pMixedCtx->rip = u64Val;
6104 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6105 }
6106 return rc;
6107}
6108
6109
6110/**
6111 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6112 *
6113 * @returns VBox status code.
6114 * @param pVCpu The cross context virtual CPU structure.
6115 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6116 * out-of-sync. Make sure to update the required fields
6117 * before using them.
6118 *
6119 * @remarks No-long-jump zone!!!
6120 */
6121static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6122{
6123 int rc = VINF_SUCCESS;
6124 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6125 {
6126 uint64_t u64Val = 0;
6127 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6128 AssertRCReturn(rc, rc);
6129
6130 pMixedCtx->rsp = u64Val;
6131 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6132 }
6133 return rc;
6134}
6135
6136
6137/**
6138 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6139 *
6140 * @returns VBox status code.
6141 * @param pVCpu The cross context virtual CPU structure.
6142 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6143 * out-of-sync. Make sure to update the required fields
6144 * before using them.
6145 *
6146 * @remarks No-long-jump zone!!!
6147 */
6148static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6149{
6150 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6151 {
6152 uint32_t uVal = 0;
6153 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6154 AssertRCReturn(rc, rc);
6155
6156 pMixedCtx->eflags.u32 = uVal;
6157 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6158 {
6159 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6160 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6161
6162 pMixedCtx->eflags.Bits.u1VM = 0;
6163 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6164 }
6165
6166 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6167 }
6168 return VINF_SUCCESS;
6169}
6170
6171
6172/**
6173 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6174 * guest-CPU context.
6175 */
6176DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6177{
6178 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6179 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6180 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6181 return rc;
6182}
6183
6184
6185/**
6186 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6187 * from the guest-state area in the VMCS.
6188 *
6189 * @param pVCpu The cross context virtual CPU structure.
6190 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6191 * out-of-sync. Make sure to update the required fields
6192 * before using them.
6193 *
6194 * @remarks No-long-jump zone!!!
6195 */
6196static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6197{
6198 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6199 {
6200 uint32_t uIntrState = 0;
6201 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6202 AssertRC(rc);
6203
6204 if (!uIntrState)
6205 {
6206 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6207 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6208
6209 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6210 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6211 }
6212 else
6213 {
6214 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6215 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6216 {
6217 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6218 AssertRC(rc);
6219 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6220 AssertRC(rc);
6221
6222 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6223 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6224 }
6225 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6226 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6227
6228 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6229 {
6230 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6231 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6232 }
6233 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6234 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6235 }
6236
6237 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6238 }
6239}
6240
6241
6242/**
6243 * Saves the guest's activity state.
6244 *
6245 * @returns VBox status code.
6246 * @param pVCpu The cross context virtual CPU structure.
6247 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6248 * out-of-sync. Make sure to update the required fields
6249 * before using them.
6250 *
6251 * @remarks No-long-jump zone!!!
6252 */
6253static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6254{
6255 NOREF(pMixedCtx);
6256 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6257 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6258 return VINF_SUCCESS;
6259}
6260
6261
6262/**
6263 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6264 * the current VMCS into the guest-CPU context.
6265 *
6266 * @returns VBox status code.
6267 * @param pVCpu The cross context virtual CPU structure.
6268 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6269 * out-of-sync. Make sure to update the required fields
6270 * before using them.
6271 *
6272 * @remarks No-long-jump zone!!!
6273 */
6274static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6275{
6276 int rc = VINF_SUCCESS;
6277 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6278 {
6279 uint32_t u32Val = 0;
6280 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6281 pMixedCtx->SysEnter.cs = u32Val;
6282 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6283 }
6284
6285 uint64_t u64Val = 0;
6286 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6287 {
6288 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6289 pMixedCtx->SysEnter.eip = u64Val;
6290 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6291 }
6292 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6293 {
6294 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6295 pMixedCtx->SysEnter.esp = u64Val;
6296 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6297 }
6298 return rc;
6299}
6300
6301
6302/**
6303 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6304 * the CPU back into the guest-CPU context.
6305 *
6306 * @returns VBox status code.
6307 * @param pVCpu The cross context virtual CPU structure.
6308 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6309 * out-of-sync. Make sure to update the required fields
6310 * before using them.
6311 *
6312 * @remarks No-long-jump zone!!!
6313 */
6314static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6315{
6316 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6317 VMMRZCallRing3Disable(pVCpu);
6318 HM_DISABLE_PREEMPT();
6319
6320 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6321 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6322 {
6323 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6324 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6325 }
6326
6327 HM_RESTORE_PREEMPT();
6328 VMMRZCallRing3Enable(pVCpu);
6329
6330 return VINF_SUCCESS;
6331}
6332
6333
6334/**
6335 * Saves the auto load/store'd guest MSRs from the current VMCS into
6336 * the guest-CPU context.
6337 *
6338 * @returns VBox status code.
6339 * @param pVCpu The cross context virtual CPU structure.
6340 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6341 * out-of-sync. Make sure to update the required fields
6342 * before using them.
6343 *
6344 * @remarks No-long-jump zone!!!
6345 */
6346static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6347{
6348 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6349 return VINF_SUCCESS;
6350
6351 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6352 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6353 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6354 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6355 {
6356 switch (pMsr->u32Msr)
6357 {
6358 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6359 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6360 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6361 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6362 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6363 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6364 break;
6365
6366 default:
6367 {
6368 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6369 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6370 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6371 }
6372 }
6373 }
6374
6375 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6376 return VINF_SUCCESS;
6377}
6378
6379
6380/**
6381 * Saves the guest control registers from the current VMCS into the guest-CPU
6382 * context.
6383 *
6384 * @returns VBox status code.
6385 * @param pVCpu The cross context virtual CPU structure.
6386 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6387 * out-of-sync. Make sure to update the required fields
6388 * before using them.
6389 *
6390 * @remarks No-long-jump zone!!!
6391 */
6392static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6393{
6394 /* Guest CR0. Guest FPU. */
6395 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6396 AssertRCReturn(rc, rc);
6397
6398 /* Guest CR4. */
6399 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6400 AssertRCReturn(rc, rc);
6401
6402 /* Guest CR2 - updated always during the world-switch or in #PF. */
6403 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6404 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6405 {
6406 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6407 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6408
6409 PVM pVM = pVCpu->CTX_SUFF(pVM);
6410 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6411 || ( pVM->hm.s.fNestedPaging
6412 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6413 {
6414 uint64_t u64Val = 0;
6415 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6416 if (pMixedCtx->cr3 != u64Val)
6417 {
6418 CPUMSetGuestCR3(pVCpu, u64Val);
6419 if (VMMRZCallRing3IsEnabled(pVCpu))
6420 {
6421 PGMUpdateCR3(pVCpu, u64Val);
6422 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6423 }
6424 else
6425 {
6426 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6427 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6428 }
6429 }
6430
6431 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6432 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6433 {
6434 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6435 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6436 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6437 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6438 AssertRCReturn(rc, rc);
6439
6440 if (VMMRZCallRing3IsEnabled(pVCpu))
6441 {
6442 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6443 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6444 }
6445 else
6446 {
6447 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6448 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6449 }
6450 }
6451 }
6452
6453 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6454 }
6455
6456 /*
6457 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6458 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6459 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6460 *
6461 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6462 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6463 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6464 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6465 *
6466 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6467 */
6468 if (VMMRZCallRing3IsEnabled(pVCpu))
6469 {
6470 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6471 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6472
6473 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6474 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6475
6476 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6477 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6478 }
6479
6480 return rc;
6481}
6482
6483
6484/**
6485 * Reads a guest segment register from the current VMCS into the guest-CPU
6486 * context.
6487 *
6488 * @returns VBox status code.
6489 * @param pVCpu The cross context virtual CPU structure.
6490 * @param idxSel Index of the selector in the VMCS.
6491 * @param idxLimit Index of the segment limit in the VMCS.
6492 * @param idxBase Index of the segment base in the VMCS.
6493 * @param idxAccess Index of the access rights of the segment in the VMCS.
6494 * @param pSelReg Pointer to the segment selector.
6495 *
6496 * @remarks No-long-jump zone!!!
6497 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6498 * macro as that takes care of whether to read from the VMCS cache or
6499 * not.
6500 */
6501DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6502 PCPUMSELREG pSelReg)
6503{
6504 NOREF(pVCpu);
6505
6506 uint32_t u32Val = 0;
6507 int rc = VMXReadVmcs32(idxSel, &u32Val);
6508 AssertRCReturn(rc, rc);
6509 pSelReg->Sel = (uint16_t)u32Val;
6510 pSelReg->ValidSel = (uint16_t)u32Val;
6511 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6512
6513 rc = VMXReadVmcs32(idxLimit, &u32Val);
6514 AssertRCReturn(rc, rc);
6515 pSelReg->u32Limit = u32Val;
6516
6517 uint64_t u64Val = 0;
6518 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6519 AssertRCReturn(rc, rc);
6520 pSelReg->u64Base = u64Val;
6521
6522 rc = VMXReadVmcs32(idxAccess, &u32Val);
6523 AssertRCReturn(rc, rc);
6524 pSelReg->Attr.u = u32Val;
6525
6526 /*
6527 * If VT-x marks the segment as unusable, most other bits remain undefined:
6528 * - For CS the L, D and G bits have meaning.
6529 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6530 * - For the remaining data segments no bits are defined.
6531 *
6532 * The present bit and the unusable bit has been observed to be set at the
6533 * same time (the selector was supposed to be invalid as we started executing
6534 * a V8086 interrupt in ring-0).
6535 *
6536 * What should be important for the rest of the VBox code, is that the P bit is
6537 * cleared. Some of the other VBox code recognizes the unusable bit, but
6538 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6539 * safe side here, we'll strip off P and other bits we don't care about. If
6540 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6541 *
6542 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6543 */
6544 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6545 {
6546 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6547
6548 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6549 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6550 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6551
6552 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6553#ifdef DEBUG_bird
6554 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6555 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6556 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6557#endif
6558 }
6559 return VINF_SUCCESS;
6560}
6561
6562
6563#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6564# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6565 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6566 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6567#else
6568# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6569 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6570 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6571#endif
6572
6573
6574/**
6575 * Saves the guest segment registers from the current VMCS into the guest-CPU
6576 * context.
6577 *
6578 * @returns VBox status code.
6579 * @param pVCpu The cross context virtual CPU structure.
6580 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6581 * out-of-sync. Make sure to update the required fields
6582 * before using them.
6583 *
6584 * @remarks No-long-jump zone!!!
6585 */
6586static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6587{
6588 /* Guest segment registers. */
6589 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6590 {
6591 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6592 AssertRCReturn(rc, rc);
6593
6594 rc = VMXLOCAL_READ_SEG(CS, cs);
6595 rc |= VMXLOCAL_READ_SEG(SS, ss);
6596 rc |= VMXLOCAL_READ_SEG(DS, ds);
6597 rc |= VMXLOCAL_READ_SEG(ES, es);
6598 rc |= VMXLOCAL_READ_SEG(FS, fs);
6599 rc |= VMXLOCAL_READ_SEG(GS, gs);
6600 AssertRCReturn(rc, rc);
6601
6602 /* Restore segment attributes for real-on-v86 mode hack. */
6603 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6604 {
6605 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6606 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6607 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6608 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6609 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6610 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6611 }
6612 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6613 }
6614
6615 return VINF_SUCCESS;
6616}
6617
6618
6619/**
6620 * Saves the guest descriptor table registers and task register from the current
6621 * VMCS into the guest-CPU context.
6622 *
6623 * @returns VBox status code.
6624 * @param pVCpu The cross context virtual CPU structure.
6625 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6626 * out-of-sync. Make sure to update the required fields
6627 * before using them.
6628 *
6629 * @remarks No-long-jump zone!!!
6630 */
6631static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6632{
6633 int rc = VINF_SUCCESS;
6634
6635 /* Guest LDTR. */
6636 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6637 {
6638 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6639 AssertRCReturn(rc, rc);
6640 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6641 }
6642
6643 /* Guest GDTR. */
6644 uint64_t u64Val = 0;
6645 uint32_t u32Val = 0;
6646 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6647 {
6648 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6649 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6650 pMixedCtx->gdtr.pGdt = u64Val;
6651 pMixedCtx->gdtr.cbGdt = u32Val;
6652 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6653 }
6654
6655 /* Guest IDTR. */
6656 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6657 {
6658 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6659 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6660 pMixedCtx->idtr.pIdt = u64Val;
6661 pMixedCtx->idtr.cbIdt = u32Val;
6662 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6663 }
6664
6665 /* Guest TR. */
6666 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6667 {
6668 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6669 AssertRCReturn(rc, rc);
6670
6671 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6672 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6673 {
6674 rc = VMXLOCAL_READ_SEG(TR, tr);
6675 AssertRCReturn(rc, rc);
6676 }
6677 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6678 }
6679 return rc;
6680}
6681
6682#undef VMXLOCAL_READ_SEG
6683
6684
6685/**
6686 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6687 * context.
6688 *
6689 * @returns VBox status code.
6690 * @param pVCpu The cross context virtual CPU structure.
6691 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6692 * out-of-sync. Make sure to update the required fields
6693 * before using them.
6694 *
6695 * @remarks No-long-jump zone!!!
6696 */
6697static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6698{
6699 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6700 {
6701 if (!pVCpu->hm.s.fUsingHyperDR7)
6702 {
6703 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6704 uint32_t u32Val;
6705 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6706 pMixedCtx->dr[7] = u32Val;
6707 }
6708
6709 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6710 }
6711 return VINF_SUCCESS;
6712}
6713
6714
6715/**
6716 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6717 *
6718 * @returns VBox status code.
6719 * @param pVCpu The cross context virtual CPU structure.
6720 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6721 * out-of-sync. Make sure to update the required fields
6722 * before using them.
6723 *
6724 * @remarks No-long-jump zone!!!
6725 */
6726static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6727{
6728 NOREF(pMixedCtx);
6729
6730 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6731 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6732 return VINF_SUCCESS;
6733}
6734
6735
6736/**
6737 * Saves the entire guest state from the currently active VMCS into the
6738 * guest-CPU context.
6739 *
6740 * This essentially VMREADs all guest-data.
6741 *
6742 * @returns VBox status code.
6743 * @param pVCpu The cross context virtual CPU structure.
6744 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6745 * out-of-sync. Make sure to update the required fields
6746 * before using them.
6747 */
6748static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6749{
6750 Assert(pVCpu);
6751 Assert(pMixedCtx);
6752
6753 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6754 return VINF_SUCCESS;
6755
6756 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6757 again on the ring-3 callback path, there is no real need to. */
6758 if (VMMRZCallRing3IsEnabled(pVCpu))
6759 VMMR0LogFlushDisable(pVCpu);
6760 else
6761 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6762 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6763
6764 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6765 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6766
6767 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6768 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6769
6770 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6771 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6772
6773 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6774 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6775
6776 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6777 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6778
6779 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6780 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6781
6782 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6783 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6784
6785 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6786 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6787
6788 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6789 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6790
6791 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6792 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6793
6794 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6795 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6796 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6797
6798 if (VMMRZCallRing3IsEnabled(pVCpu))
6799 VMMR0LogFlushEnable(pVCpu);
6800
6801 return VINF_SUCCESS;
6802}
6803
6804
6805/**
6806 * Saves basic guest registers needed for IEM instruction execution.
6807 *
6808 * @returns VBox status code (OR-able).
6809 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6810 * @param pMixedCtx Pointer to the CPU context of the guest.
6811 * @param fMemory Whether the instruction being executed operates on
6812 * memory or not. Only CR0 is synced up if clear.
6813 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6814 */
6815static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6816{
6817 /*
6818 * We assume all general purpose registers other than RSP are available.
6819 *
6820 * - RIP is a must, as it will be incremented or otherwise changed.
6821 * - RFLAGS are always required to figure the CPL.
6822 * - RSP isn't always required, however it's a GPR, so frequently required.
6823 * - SS and CS are the only segment register needed if IEM doesn't do memory
6824 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6825 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6826 * be required for memory accesses.
6827 *
6828 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6829 */
6830 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6831 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6832 if (fNeedRsp)
6833 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6834 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6835 if (!fMemory)
6836 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6837 else
6838 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6839 AssertRCReturn(rc, rc);
6840 return rc;
6841}
6842
6843
6844/**
6845 * Ensures that we've got a complete basic guest-context.
6846 *
6847 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6848 * is for the interpreter.
6849 *
6850 * @returns VBox status code.
6851 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6852 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6853 * needing to be synced in.
6854 * @thread EMT(pVCpu)
6855 */
6856VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6857{
6858 /* Note! Since this is only applicable to VT-x, the implementation is placed
6859 in the VT-x part of the sources instead of the generic stuff. */
6860 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6861 {
6862 /* For now, imply that the caller might change everything too. */
6863 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6864 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6865 }
6866 return VINF_SUCCESS;
6867}
6868
6869
6870/**
6871 * Check per-VM and per-VCPU force flag actions that require us to go back to
6872 * ring-3 for one reason or another.
6873 *
6874 * @returns Strict VBox status code (i.e. informational status codes too)
6875 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6876 * ring-3.
6877 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6878 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6879 * interrupts)
6880 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6881 * all EMTs to be in ring-3.
6882 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6883 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6884 * to the EM loop.
6885 *
6886 * @param pVM The cross context VM structure.
6887 * @param pVCpu The cross context virtual CPU structure.
6888 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6889 * out-of-sync. Make sure to update the required fields
6890 * before using them.
6891 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6892 */
6893static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6894{
6895 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6896
6897 /*
6898 * Anything pending? Should be more likely than not if we're doing a good job.
6899 */
6900 if ( !fStepping
6901 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6902 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6903 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6904 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6905 return VINF_SUCCESS;
6906
6907 /* We need the control registers now, make sure the guest-CPU context is updated. */
6908 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6909 AssertRCReturn(rc3, rc3);
6910
6911 /* Pending HM CR3 sync. */
6912 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6913 {
6914 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6915 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6916 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6917 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6918 }
6919
6920 /* Pending HM PAE PDPEs. */
6921 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6922 {
6923 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6924 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6925 }
6926
6927 /* Pending PGM C3 sync. */
6928 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6929 {
6930 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6931 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6932 if (rcStrict2 != VINF_SUCCESS)
6933 {
6934 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6935 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6936 return rcStrict2;
6937 }
6938 }
6939
6940 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6941 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6942 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6943 {
6944 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6945 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6946 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6947 return rc2;
6948 }
6949
6950 /* Pending VM request packets, such as hardware interrupts. */
6951 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6952 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6953 {
6954 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6955 return VINF_EM_PENDING_REQUEST;
6956 }
6957
6958 /* Pending PGM pool flushes. */
6959 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6960 {
6961 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6962 return VINF_PGM_POOL_FLUSH_PENDING;
6963 }
6964
6965 /* Pending DMA requests. */
6966 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6967 {
6968 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6969 return VINF_EM_RAW_TO_R3;
6970 }
6971
6972 return VINF_SUCCESS;
6973}
6974
6975
6976/**
6977 * Converts any TRPM trap into a pending HM event. This is typically used when
6978 * entering from ring-3 (not longjmp returns).
6979 *
6980 * @param pVCpu The cross context virtual CPU structure.
6981 */
6982static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6983{
6984 Assert(TRPMHasTrap(pVCpu));
6985 Assert(!pVCpu->hm.s.Event.fPending);
6986
6987 uint8_t uVector;
6988 TRPMEVENT enmTrpmEvent;
6989 RTGCUINT uErrCode;
6990 RTGCUINTPTR GCPtrFaultAddress;
6991 uint8_t cbInstr;
6992
6993 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6994 AssertRC(rc);
6995
6996 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6997 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6998 if (enmTrpmEvent == TRPM_TRAP)
6999 {
7000 switch (uVector)
7001 {
7002 case X86_XCPT_NMI:
7003 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7004 break;
7005
7006 case X86_XCPT_BP:
7007 case X86_XCPT_OF:
7008 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7009 break;
7010
7011 case X86_XCPT_PF:
7012 case X86_XCPT_DF:
7013 case X86_XCPT_TS:
7014 case X86_XCPT_NP:
7015 case X86_XCPT_SS:
7016 case X86_XCPT_GP:
7017 case X86_XCPT_AC:
7018 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7019 /* no break! */
7020 default:
7021 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7022 break;
7023 }
7024 }
7025 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7026 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7027 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7028 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7029 else
7030 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7031
7032 rc = TRPMResetTrap(pVCpu);
7033 AssertRC(rc);
7034 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7035 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7036
7037 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7038}
7039
7040
7041/**
7042 * Converts the pending HM event into a TRPM trap.
7043 *
7044 * @param pVCpu The cross context virtual CPU structure.
7045 */
7046static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7047{
7048 Assert(pVCpu->hm.s.Event.fPending);
7049
7050 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7051 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7052 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7053 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7054
7055 /* If a trap was already pending, we did something wrong! */
7056 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7057
7058 TRPMEVENT enmTrapType;
7059 switch (uVectorType)
7060 {
7061 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7062 enmTrapType = TRPM_HARDWARE_INT;
7063 break;
7064
7065 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7066 enmTrapType = TRPM_SOFTWARE_INT;
7067 break;
7068
7069 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7070 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7071 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7072 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7073 enmTrapType = TRPM_TRAP;
7074 break;
7075
7076 default:
7077 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7078 enmTrapType = TRPM_32BIT_HACK;
7079 break;
7080 }
7081
7082 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7083
7084 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7085 AssertRC(rc);
7086
7087 if (fErrorCodeValid)
7088 TRPMSetErrorCode(pVCpu, uErrorCode);
7089
7090 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7091 && uVector == X86_XCPT_PF)
7092 {
7093 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7094 }
7095 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7096 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7097 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7098 {
7099 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7100 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7101 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7102 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7103 }
7104
7105 /* Clear any pending events from the VMCS. */
7106 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7107 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7108
7109 /* We're now done converting the pending event. */
7110 pVCpu->hm.s.Event.fPending = false;
7111}
7112
7113
7114/**
7115 * Does the necessary state syncing before returning to ring-3 for any reason
7116 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7117 *
7118 * @returns VBox status code.
7119 * @param pVCpu The cross context virtual CPU structure.
7120 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7121 * be out-of-sync. Make sure to update the required
7122 * fields before using them.
7123 * @param fSaveGuestState Whether to save the guest state or not.
7124 *
7125 * @remarks No-long-jmp zone!!!
7126 */
7127static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7128{
7129 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7130 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7131
7132 RTCPUID idCpu = RTMpCpuId();
7133 Log4Func(("HostCpuId=%u\n", idCpu));
7134
7135 /*
7136 * !!! IMPORTANT !!!
7137 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7138 */
7139
7140 /* Save the guest state if necessary. */
7141 if ( fSaveGuestState
7142 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7143 {
7144 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7145 AssertRCReturn(rc, rc);
7146 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7147 }
7148
7149 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7150 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7151 {
7152 if (fSaveGuestState)
7153 {
7154 /* We shouldn't reload CR0 without saving it first. */
7155 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7156 AssertRCReturn(rc, rc);
7157 }
7158 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7159 }
7160
7161 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7162#ifdef VBOX_STRICT
7163 if (CPUMIsHyperDebugStateActive(pVCpu))
7164 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7165#endif
7166 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7167 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7168 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7169 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7170
7171#if HC_ARCH_BITS == 64
7172 /* Restore host-state bits that VT-x only restores partially. */
7173 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7174 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7175 {
7176 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7177 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7178 }
7179 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7180#endif
7181
7182 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7183 if (pVCpu->hm.s.vmx.fLazyMsrs)
7184 {
7185 /* We shouldn't reload the guest MSRs without saving it first. */
7186 if (!fSaveGuestState)
7187 {
7188 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7189 AssertRCReturn(rc, rc);
7190 }
7191 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7192 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7193 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7194 }
7195
7196 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7197 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7198
7199 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7200 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7201 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7202 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7203 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7204 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7205 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7206 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7207
7208 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7209
7210 /** @todo This partially defeats the purpose of having preemption hooks.
7211 * The problem is, deregistering the hooks should be moved to a place that
7212 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7213 * context.
7214 */
7215 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7216 {
7217 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7218 AssertRCReturn(rc, rc);
7219
7220 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7221 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7222 }
7223 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7224 NOREF(idCpu);
7225
7226 return VINF_SUCCESS;
7227}
7228
7229
7230/**
7231 * Leaves the VT-x session.
7232 *
7233 * @returns VBox status code.
7234 * @param pVCpu The cross context virtual CPU structure.
7235 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7236 * out-of-sync. Make sure to update the required fields
7237 * before using them.
7238 *
7239 * @remarks No-long-jmp zone!!!
7240 */
7241DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7242{
7243 HM_DISABLE_PREEMPT();
7244 HMVMX_ASSERT_CPU_SAFE();
7245 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7246 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7247
7248 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7249 and done this from the VMXR0ThreadCtxCallback(). */
7250 if (!pVCpu->hm.s.fLeaveDone)
7251 {
7252 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7253 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7254 pVCpu->hm.s.fLeaveDone = true;
7255 }
7256 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7257
7258 /*
7259 * !!! IMPORTANT !!!
7260 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7261 */
7262
7263 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7264 /** @todo Deregistering here means we need to VMCLEAR always
7265 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7266 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7267 VMMR0ThreadCtxHookDisable(pVCpu);
7268
7269 /* Leave HM context. This takes care of local init (term). */
7270 int rc = HMR0LeaveCpu(pVCpu);
7271
7272 HM_RESTORE_PREEMPT();
7273 return rc;
7274}
7275
7276
7277/**
7278 * Does the necessary state syncing before doing a longjmp to ring-3.
7279 *
7280 * @returns VBox status code.
7281 * @param pVCpu The cross context virtual CPU structure.
7282 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7283 * out-of-sync. Make sure to update the required fields
7284 * before using them.
7285 *
7286 * @remarks No-long-jmp zone!!!
7287 */
7288DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7289{
7290 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7291}
7292
7293
7294/**
7295 * Take necessary actions before going back to ring-3.
7296 *
7297 * An action requires us to go back to ring-3. This function does the necessary
7298 * steps before we can safely return to ring-3. This is not the same as longjmps
7299 * to ring-3, this is voluntary and prepares the guest so it may continue
7300 * executing outside HM (recompiler/IEM).
7301 *
7302 * @returns VBox status code.
7303 * @param pVM The cross context VM structure.
7304 * @param pVCpu The cross context virtual CPU structure.
7305 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7306 * out-of-sync. Make sure to update the required fields
7307 * before using them.
7308 * @param rcExit The reason for exiting to ring-3. Can be
7309 * VINF_VMM_UNKNOWN_RING3_CALL.
7310 */
7311static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7312{
7313 Assert(pVM);
7314 Assert(pVCpu);
7315 Assert(pMixedCtx);
7316 HMVMX_ASSERT_PREEMPT_SAFE();
7317
7318 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7319 {
7320 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7321 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7322 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7323 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7324 }
7325
7326 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7327 VMMRZCallRing3Disable(pVCpu);
7328 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7329
7330 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7331 if (pVCpu->hm.s.Event.fPending)
7332 {
7333 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7334 Assert(!pVCpu->hm.s.Event.fPending);
7335 }
7336
7337 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7338 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7339
7340 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7341 and if we're injecting an event we should have a TRPM trap pending. */
7342 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7343#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7344 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7345#endif
7346
7347 /* Save guest state and restore host state bits. */
7348 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7349 AssertRCReturn(rc, rc);
7350 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7351 /* Thread-context hooks are unregistered at this point!!! */
7352
7353 /* Sync recompiler state. */
7354 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7355 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7356 | CPUM_CHANGED_LDTR
7357 | CPUM_CHANGED_GDTR
7358 | CPUM_CHANGED_IDTR
7359 | CPUM_CHANGED_TR
7360 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7361 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7362 if ( pVM->hm.s.fNestedPaging
7363 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7364 {
7365 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7366 }
7367
7368 Assert(!pVCpu->hm.s.fClearTrapFlag);
7369
7370 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7371 if (rcExit != VINF_EM_RAW_INTERRUPT)
7372 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7373
7374 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7375
7376 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7377 VMMRZCallRing3RemoveNotification(pVCpu);
7378 VMMRZCallRing3Enable(pVCpu);
7379
7380 return rc;
7381}
7382
7383
7384/**
7385 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7386 * longjump to ring-3 and possibly get preempted.
7387 *
7388 * @returns VBox status code.
7389 * @param pVCpu The cross context virtual CPU structure.
7390 * @param enmOperation The operation causing the ring-3 longjump.
7391 * @param pvUser Opaque pointer to the guest-CPU context. The data
7392 * may be out-of-sync. Make sure to update the required
7393 * fields before using them.
7394 */
7395static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7396{
7397 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7398 {
7399 /*
7400 * !!! IMPORTANT !!!
7401 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7402 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7403 */
7404 VMMRZCallRing3RemoveNotification(pVCpu);
7405 VMMRZCallRing3Disable(pVCpu);
7406 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7407 RTThreadPreemptDisable(&PreemptState);
7408
7409 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7410 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7411
7412#if HC_ARCH_BITS == 64
7413 /* Restore host-state bits that VT-x only restores partially. */
7414 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7415 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7416 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7417 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7418#endif
7419 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7420 if (pVCpu->hm.s.vmx.fLazyMsrs)
7421 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7422
7423 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7424 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7425 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7426 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7427 {
7428 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7429 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7430 }
7431
7432 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7433 VMMR0ThreadCtxHookDisable(pVCpu);
7434 HMR0LeaveCpu(pVCpu);
7435 RTThreadPreemptRestore(&PreemptState);
7436 return VINF_SUCCESS;
7437 }
7438
7439 Assert(pVCpu);
7440 Assert(pvUser);
7441 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7442 HMVMX_ASSERT_PREEMPT_SAFE();
7443
7444 VMMRZCallRing3Disable(pVCpu);
7445 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7446
7447 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7448 enmOperation));
7449
7450 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7451 AssertRCReturn(rc, rc);
7452
7453 VMMRZCallRing3Enable(pVCpu);
7454 return VINF_SUCCESS;
7455}
7456
7457
7458/**
7459 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7460 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7461 *
7462 * @param pVCpu The cross context virtual CPU structure.
7463 */
7464DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7465{
7466 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7467 {
7468 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7469 {
7470 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7471 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7472 AssertRC(rc);
7473 Log4(("Setup interrupt-window exiting\n"));
7474 }
7475 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7476}
7477
7478
7479/**
7480 * Clears the interrupt-window exiting control in the VMCS.
7481 *
7482 * @param pVCpu The cross context virtual CPU structure.
7483 */
7484DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7485{
7486 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7487 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7488 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7489 AssertRC(rc);
7490 Log4(("Cleared interrupt-window exiting\n"));
7491}
7492
7493
7494/**
7495 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7496 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7497 *
7498 * @param pVCpu The cross context virtual CPU structure.
7499 */
7500DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7501{
7502 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7503 {
7504 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7505 {
7506 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7507 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7508 AssertRC(rc);
7509 Log4(("Setup NMI-window exiting\n"));
7510 }
7511 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7512}
7513
7514
7515/**
7516 * Clears the NMI-window exiting control in the VMCS.
7517 *
7518 * @param pVCpu The cross context virtual CPU structure.
7519 */
7520DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7521{
7522 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7523 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7524 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7525 AssertRC(rc);
7526 Log4(("Cleared NMI-window exiting\n"));
7527}
7528
7529
7530/**
7531 * Evaluates the event to be delivered to the guest and sets it as the pending
7532 * event.
7533 *
7534 * @returns The VT-x guest-interruptibility state.
7535 * @param pVCpu The cross context virtual CPU structure.
7536 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7537 * out-of-sync. Make sure to update the required fields
7538 * before using them.
7539 */
7540static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7541{
7542 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7543 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7544 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7545 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7546 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7547
7548 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7549 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7550 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7551 Assert(!TRPMHasTrap(pVCpu));
7552
7553 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7554 APICUpdatePendingInterrupts(pVCpu);
7555
7556 /*
7557 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7558 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7559 */
7560 /** @todo SMI. SMIs take priority over NMIs. */
7561 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7562 {
7563 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7564 if ( !pVCpu->hm.s.Event.fPending
7565 && !fBlockNmi
7566 && !fBlockSti
7567 && !fBlockMovSS)
7568 {
7569 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7570 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7571 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7572
7573 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7574 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7575 }
7576 else
7577 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7578 }
7579 /*
7580 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7581 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7582 */
7583 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7584 && !pVCpu->hm.s.fSingleInstruction)
7585 {
7586 Assert(!DBGFIsStepping(pVCpu));
7587 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7588 AssertRC(rc);
7589 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7590 if ( !pVCpu->hm.s.Event.fPending
7591 && !fBlockInt
7592 && !fBlockSti
7593 && !fBlockMovSS)
7594 {
7595 uint8_t u8Interrupt;
7596 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7597 if (RT_SUCCESS(rc))
7598 {
7599 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7600 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7601 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7602
7603 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7604 }
7605 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7606 {
7607 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7608 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7609 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7610
7611 /*
7612 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7613 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7614 * need to re-set this force-flag here.
7615 */
7616 }
7617 else
7618 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7619 }
7620 else
7621 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7622 }
7623
7624 return uIntrState;
7625}
7626
7627
7628/**
7629 * Sets a pending-debug exception to be delivered to the guest if the guest is
7630 * single-stepping in the VMCS.
7631 *
7632 * @param pVCpu The cross context virtual CPU structure.
7633 */
7634DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7635{
7636 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7637 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7638 AssertRC(rc);
7639}
7640
7641
7642/**
7643 * Injects any pending events into the guest if the guest is in a state to
7644 * receive them.
7645 *
7646 * @returns Strict VBox status code (i.e. informational status codes too).
7647 * @param pVCpu The cross context virtual CPU structure.
7648 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7649 * out-of-sync. Make sure to update the required fields
7650 * before using them.
7651 * @param uIntrState The VT-x guest-interruptibility state.
7652 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7653 * return VINF_EM_DBG_STEPPED if the event was
7654 * dispatched directly.
7655 */
7656static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7657{
7658 HMVMX_ASSERT_PREEMPT_SAFE();
7659 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7660
7661 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7662 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7663
7664 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7665 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7666 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7667 Assert(!TRPMHasTrap(pVCpu));
7668
7669 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7670 if (pVCpu->hm.s.Event.fPending)
7671 {
7672 /*
7673 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7674 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7675 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7676 *
7677 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7678 */
7679 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7680#ifdef VBOX_STRICT
7681 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7682 {
7683 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7684 Assert(!fBlockInt);
7685 Assert(!fBlockSti);
7686 Assert(!fBlockMovSS);
7687 }
7688 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7689 {
7690 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7691 Assert(!fBlockSti);
7692 Assert(!fBlockMovSS);
7693 Assert(!fBlockNmi);
7694 }
7695#endif
7696 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7697 (uint8_t)uIntType));
7698 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7699 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7700 fStepping, &uIntrState);
7701 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7702
7703 /* Update the interruptibility-state as it could have been changed by
7704 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7705 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7706 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7707
7708 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7709 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7710 else
7711 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7712 }
7713
7714 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7715 if ( fBlockSti
7716 || fBlockMovSS)
7717 {
7718 if (!pVCpu->hm.s.fSingleInstruction)
7719 {
7720 /*
7721 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7722 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7723 * See Intel spec. 27.3.4 "Saving Non-Register State".
7724 */
7725 Assert(!DBGFIsStepping(pVCpu));
7726 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7727 AssertRCReturn(rc2, rc2);
7728 if (pMixedCtx->eflags.Bits.u1TF)
7729 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7730 }
7731 else if (pMixedCtx->eflags.Bits.u1TF)
7732 {
7733 /*
7734 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7735 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7736 */
7737 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7738 uIntrState = 0;
7739 }
7740 }
7741
7742 /*
7743 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7744 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7745 */
7746 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7747 AssertRC(rc2);
7748
7749 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7750 NOREF(fBlockMovSS); NOREF(fBlockSti);
7751 return rcStrict;
7752}
7753
7754
7755/**
7756 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7757 *
7758 * @param pVCpu The cross context virtual CPU structure.
7759 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7760 * out-of-sync. Make sure to update the required fields
7761 * before using them.
7762 */
7763DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7764{
7765 NOREF(pMixedCtx);
7766 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7767 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7768}
7769
7770
7771/**
7772 * Injects a double-fault (\#DF) exception into the VM.
7773 *
7774 * @returns Strict VBox status code (i.e. informational status codes too).
7775 * @param pVCpu The cross context virtual CPU structure.
7776 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7777 * out-of-sync. Make sure to update the required fields
7778 * before using them.
7779 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7780 * and should return VINF_EM_DBG_STEPPED if the event
7781 * is injected directly (register modified by us, not
7782 * by hardware on VM-entry).
7783 * @param puIntrState Pointer to the current guest interruptibility-state.
7784 * This interruptibility-state will be updated if
7785 * necessary. This cannot not be NULL.
7786 */
7787DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7788{
7789 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7790 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7791 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7792 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7793 fStepping, puIntrState);
7794}
7795
7796
7797/**
7798 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7799 *
7800 * @param pVCpu The cross context virtual CPU structure.
7801 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7802 * out-of-sync. Make sure to update the required fields
7803 * before using them.
7804 */
7805DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7806{
7807 NOREF(pMixedCtx);
7808 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7809 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7810 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7811}
7812
7813
7814/**
7815 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7816 *
7817 * @param pVCpu The cross context virtual CPU structure.
7818 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7819 * out-of-sync. Make sure to update the required fields
7820 * before using them.
7821 * @param cbInstr The value of RIP that is to be pushed on the guest
7822 * stack.
7823 */
7824DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7825{
7826 NOREF(pMixedCtx);
7827 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7828 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7829 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7830}
7831
7832
7833/**
7834 * Injects a general-protection (\#GP) fault into the VM.
7835 *
7836 * @returns Strict VBox status code (i.e. informational status codes too).
7837 * @param pVCpu The cross context virtual CPU structure.
7838 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7839 * out-of-sync. Make sure to update the required fields
7840 * before using them.
7841 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7842 * mode, i.e. in real-mode it's not valid).
7843 * @param u32ErrorCode The error code associated with the \#GP.
7844 * @param fStepping Whether we're running in
7845 * hmR0VmxRunGuestCodeStep() and should return
7846 * VINF_EM_DBG_STEPPED if the event is injected
7847 * directly (register modified by us, not by
7848 * hardware on VM-entry).
7849 * @param puIntrState Pointer to the current guest interruptibility-state.
7850 * This interruptibility-state will be updated if
7851 * necessary. This cannot not be NULL.
7852 */
7853DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7854 bool fStepping, uint32_t *puIntrState)
7855{
7856 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7857 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7858 if (fErrorCodeValid)
7859 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7860 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7861 fStepping, puIntrState);
7862}
7863
7864
7865#if 0 /* unused */
7866/**
7867 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7868 * VM.
7869 *
7870 * @param pVCpu The cross context virtual CPU structure.
7871 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7872 * out-of-sync. Make sure to update the required fields
7873 * before using them.
7874 * @param u32ErrorCode The error code associated with the \#GP.
7875 */
7876DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7877{
7878 NOREF(pMixedCtx);
7879 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7880 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7881 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7882 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7883}
7884#endif /* unused */
7885
7886
7887/**
7888 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7889 *
7890 * @param pVCpu The cross context virtual CPU structure.
7891 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7892 * out-of-sync. Make sure to update the required fields
7893 * before using them.
7894 * @param uVector The software interrupt vector number.
7895 * @param cbInstr The value of RIP that is to be pushed on the guest
7896 * stack.
7897 */
7898DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7899{
7900 NOREF(pMixedCtx);
7901 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7902 if ( uVector == X86_XCPT_BP
7903 || uVector == X86_XCPT_OF)
7904 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7905 else
7906 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7907 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7908}
7909
7910
7911/**
7912 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7913 * stack.
7914 *
7915 * @returns Strict VBox status code (i.e. informational status codes too).
7916 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7917 * @param pVM The cross context VM structure.
7918 * @param pMixedCtx Pointer to the guest-CPU context.
7919 * @param uValue The value to push to the guest stack.
7920 */
7921DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7922{
7923 /*
7924 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7925 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7926 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7927 */
7928 if (pMixedCtx->sp == 1)
7929 return VINF_EM_RESET;
7930 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7931 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7932 AssertRC(rc);
7933 return rc;
7934}
7935
7936
7937/**
7938 * Injects an event into the guest upon VM-entry by updating the relevant fields
7939 * in the VM-entry area in the VMCS.
7940 *
7941 * @returns Strict VBox status code (i.e. informational status codes too).
7942 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7943 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7944 *
7945 * @param pVCpu The cross context virtual CPU structure.
7946 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7947 * be out-of-sync. Make sure to update the required
7948 * fields before using them.
7949 * @param u64IntInfo The VM-entry interruption-information field.
7950 * @param cbInstr The VM-entry instruction length in bytes (for
7951 * software interrupts, exceptions and privileged
7952 * software exceptions).
7953 * @param u32ErrCode The VM-entry exception error code.
7954 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7955 * @param puIntrState Pointer to the current guest interruptibility-state.
7956 * This interruptibility-state will be updated if
7957 * necessary. This cannot not be NULL.
7958 * @param fStepping Whether we're running in
7959 * hmR0VmxRunGuestCodeStep() and should return
7960 * VINF_EM_DBG_STEPPED if the event is injected
7961 * directly (register modified by us, not by
7962 * hardware on VM-entry).
7963 *
7964 * @remarks Requires CR0!
7965 */
7966static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7967 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7968 uint32_t *puIntrState)
7969{
7970 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7971 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7972 Assert(puIntrState);
7973 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7974
7975 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7976 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7977
7978#ifdef VBOX_STRICT
7979 /* Validate the error-code-valid bit for hardware exceptions. */
7980 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7981 {
7982 switch (uVector)
7983 {
7984 case X86_XCPT_PF:
7985 case X86_XCPT_DF:
7986 case X86_XCPT_TS:
7987 case X86_XCPT_NP:
7988 case X86_XCPT_SS:
7989 case X86_XCPT_GP:
7990 case X86_XCPT_AC:
7991 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7992 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7993 /* fallthru */
7994 default:
7995 break;
7996 }
7997 }
7998#endif
7999
8000 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8001 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8002 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8003
8004 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8005
8006 /* We require CR0 to check if the guest is in real-mode. */
8007 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8008 AssertRCReturn(rc, rc);
8009
8010 /*
8011 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8012 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8013 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8014 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8015 */
8016 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8017 {
8018 PVM pVM = pVCpu->CTX_SUFF(pVM);
8019 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8020 {
8021 Assert(PDMVmmDevHeapIsEnabled(pVM));
8022 Assert(pVM->hm.s.vmx.pRealModeTSS);
8023
8024 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8025 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8026 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8027 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8028 AssertRCReturn(rc, rc);
8029 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8030
8031 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8032 size_t const cbIdtEntry = sizeof(X86IDTR16);
8033 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8034 {
8035 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8036 if (uVector == X86_XCPT_DF)
8037 return VINF_EM_RESET;
8038
8039 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8040 if (uVector == X86_XCPT_GP)
8041 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8042
8043 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8044 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8045 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8046 fStepping, puIntrState);
8047 }
8048
8049 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8050 uint16_t uGuestIp = pMixedCtx->ip;
8051 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8052 {
8053 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8054 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8055 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8056 }
8057 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8058 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8059
8060 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8061 X86IDTR16 IdtEntry;
8062 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8063 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8064 AssertRCReturn(rc, rc);
8065
8066 /* Construct the stack frame for the interrupt/exception handler. */
8067 VBOXSTRICTRC rcStrict;
8068 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8069 if (rcStrict == VINF_SUCCESS)
8070 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8071 if (rcStrict == VINF_SUCCESS)
8072 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8073
8074 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8075 if (rcStrict == VINF_SUCCESS)
8076 {
8077 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8078 pMixedCtx->rip = IdtEntry.offSel;
8079 pMixedCtx->cs.Sel = IdtEntry.uSel;
8080 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8081 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8082 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8083 && uVector == X86_XCPT_PF)
8084 pMixedCtx->cr2 = GCPtrFaultAddress;
8085
8086 /* If any other guest-state bits are changed here, make sure to update
8087 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8088 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8089 | HM_CHANGED_GUEST_RIP
8090 | HM_CHANGED_GUEST_RFLAGS
8091 | HM_CHANGED_GUEST_RSP);
8092
8093 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8094 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8095 {
8096 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8097 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8098 Log4(("Clearing inhibition due to STI.\n"));
8099 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8100 }
8101 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8102 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8103
8104 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8105 it, if we are returning to ring-3 before executing guest code. */
8106 pVCpu->hm.s.Event.fPending = false;
8107
8108 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8109 if (fStepping)
8110 rcStrict = VINF_EM_DBG_STEPPED;
8111 }
8112 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8113 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8114 return rcStrict;
8115 }
8116
8117 /*
8118 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8119 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8120 */
8121 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8122 }
8123
8124 /* Validate. */
8125 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8126 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8127 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8128
8129 /* Inject. */
8130 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8131 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8132 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8133 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8134
8135 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8136 && uVector == X86_XCPT_PF)
8137 pMixedCtx->cr2 = GCPtrFaultAddress;
8138
8139 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8140 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8141
8142 AssertRCReturn(rc, rc);
8143 return VINF_SUCCESS;
8144}
8145
8146
8147/**
8148 * Clears the interrupt-window exiting control in the VMCS and if necessary
8149 * clears the current event in the VMCS as well.
8150 *
8151 * @returns VBox status code.
8152 * @param pVCpu The cross context virtual CPU structure.
8153 *
8154 * @remarks Use this function only to clear events that have not yet been
8155 * delivered to the guest but are injected in the VMCS!
8156 * @remarks No-long-jump zone!!!
8157 */
8158static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8159{
8160 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8161
8162 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8163 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8164
8165 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8166 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8167}
8168
8169
8170/**
8171 * Enters the VT-x session.
8172 *
8173 * @returns VBox status code.
8174 * @param pVM The cross context VM structure.
8175 * @param pVCpu The cross context virtual CPU structure.
8176 * @param pCpu Pointer to the CPU info struct.
8177 */
8178VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8179{
8180 AssertPtr(pVM);
8181 AssertPtr(pVCpu);
8182 Assert(pVM->hm.s.vmx.fSupported);
8183 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8184 NOREF(pCpu); NOREF(pVM);
8185
8186 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8187 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8188
8189#ifdef VBOX_STRICT
8190 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8191 RTCCUINTREG uHostCR4 = ASMGetCR4();
8192 if (!(uHostCR4 & X86_CR4_VMXE))
8193 {
8194 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8195 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8196 }
8197#endif
8198
8199 /*
8200 * Load the VCPU's VMCS as the current (and active) one.
8201 */
8202 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8203 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8204 if (RT_FAILURE(rc))
8205 return rc;
8206
8207 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8208 pVCpu->hm.s.fLeaveDone = false;
8209 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8210
8211 return VINF_SUCCESS;
8212}
8213
8214
8215/**
8216 * The thread-context callback (only on platforms which support it).
8217 *
8218 * @param enmEvent The thread-context event.
8219 * @param pVCpu The cross context virtual CPU structure.
8220 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8221 * @thread EMT(pVCpu)
8222 */
8223VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8224{
8225 NOREF(fGlobalInit);
8226
8227 switch (enmEvent)
8228 {
8229 case RTTHREADCTXEVENT_OUT:
8230 {
8231 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8232 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8233 VMCPU_ASSERT_EMT(pVCpu);
8234
8235 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8236
8237 /* No longjmps (logger flushes, locks) in this fragile context. */
8238 VMMRZCallRing3Disable(pVCpu);
8239 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8240
8241 /*
8242 * Restore host-state (FPU, debug etc.)
8243 */
8244 if (!pVCpu->hm.s.fLeaveDone)
8245 {
8246 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8247 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8248 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8249 pVCpu->hm.s.fLeaveDone = true;
8250 }
8251
8252 /* Leave HM context, takes care of local init (term). */
8253 int rc = HMR0LeaveCpu(pVCpu);
8254 AssertRC(rc); NOREF(rc);
8255
8256 /* Restore longjmp state. */
8257 VMMRZCallRing3Enable(pVCpu);
8258 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8259 break;
8260 }
8261
8262 case RTTHREADCTXEVENT_IN:
8263 {
8264 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8265 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8266 VMCPU_ASSERT_EMT(pVCpu);
8267
8268 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8269 VMMRZCallRing3Disable(pVCpu);
8270 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8271
8272 /* Initialize the bare minimum state required for HM. This takes care of
8273 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8274 int rc = HMR0EnterCpu(pVCpu);
8275 AssertRC(rc);
8276 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8277
8278 /* Load the active VMCS as the current one. */
8279 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8280 {
8281 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8282 AssertRC(rc); NOREF(rc);
8283 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8284 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8285 }
8286 pVCpu->hm.s.fLeaveDone = false;
8287
8288 /* Restore longjmp state. */
8289 VMMRZCallRing3Enable(pVCpu);
8290 break;
8291 }
8292
8293 default:
8294 break;
8295 }
8296}
8297
8298
8299/**
8300 * Saves the host state in the VMCS host-state.
8301 * Sets up the VM-exit MSR-load area.
8302 *
8303 * The CPU state will be loaded from these fields on every successful VM-exit.
8304 *
8305 * @returns VBox status code.
8306 * @param pVM The cross context VM structure.
8307 * @param pVCpu The cross context virtual CPU structure.
8308 *
8309 * @remarks No-long-jump zone!!!
8310 */
8311static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8312{
8313 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8314
8315 int rc = VINF_SUCCESS;
8316 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8317 {
8318 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8319 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8320
8321 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8322 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8323
8324 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8325 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8326
8327 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8328 }
8329 return rc;
8330}
8331
8332
8333/**
8334 * Saves the host state in the VMCS host-state.
8335 *
8336 * @returns VBox status code.
8337 * @param pVM The cross context VM structure.
8338 * @param pVCpu The cross context virtual CPU structure.
8339 *
8340 * @remarks No-long-jump zone!!!
8341 */
8342VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8343{
8344 AssertPtr(pVM);
8345 AssertPtr(pVCpu);
8346
8347 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8348
8349 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8350 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8351 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8352 return hmR0VmxSaveHostState(pVM, pVCpu);
8353}
8354
8355
8356/**
8357 * Loads the guest state into the VMCS guest-state area.
8358 *
8359 * The will typically be done before VM-entry when the guest-CPU state and the
8360 * VMCS state may potentially be out of sync.
8361 *
8362 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8363 * VM-entry controls.
8364 * Sets up the appropriate VMX non-root function to execute guest code based on
8365 * the guest CPU mode.
8366 *
8367 * @returns VBox strict status code.
8368 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8369 * without unrestricted guest access and the VMMDev is not presently
8370 * mapped (e.g. EFI32).
8371 *
8372 * @param pVM The cross context VM structure.
8373 * @param pVCpu The cross context virtual CPU structure.
8374 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8375 * out-of-sync. Make sure to update the required fields
8376 * before using them.
8377 *
8378 * @remarks No-long-jump zone!!!
8379 */
8380static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8381{
8382 AssertPtr(pVM);
8383 AssertPtr(pVCpu);
8384 AssertPtr(pMixedCtx);
8385 HMVMX_ASSERT_PREEMPT_SAFE();
8386
8387 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8388
8389 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8390
8391 /* Determine real-on-v86 mode. */
8392 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8393 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8394 && CPUMIsGuestInRealModeEx(pMixedCtx))
8395 {
8396 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8397 }
8398
8399 /*
8400 * Load the guest-state into the VMCS.
8401 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8402 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8403 */
8404 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8405 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8406
8407 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8408 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8409 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8410
8411 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8412 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8413 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8414
8415 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8416 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8417
8418 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8419 if (rcStrict == VINF_SUCCESS)
8420 { /* likely */ }
8421 else
8422 {
8423 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8424 return rcStrict;
8425 }
8426
8427 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8428 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8429 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8430
8431 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8432 determine we don't have to swap EFER after all. */
8433 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8434 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8435
8436 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8437 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8438
8439 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8440 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8441
8442 /*
8443 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8444 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8445 */
8446 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8447 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8448
8449 /* Clear any unused and reserved bits. */
8450 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8451
8452 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8453 return rc;
8454}
8455
8456
8457/**
8458 * Loads the state shared between the host and guest into the VMCS.
8459 *
8460 * @param pVM The cross context VM structure.
8461 * @param pVCpu The cross context virtual CPU structure.
8462 * @param pCtx Pointer to the guest-CPU context.
8463 *
8464 * @remarks No-long-jump zone!!!
8465 */
8466static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8467{
8468 NOREF(pVM);
8469
8470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8471 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8472
8473 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8474 {
8475 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8476 AssertRC(rc);
8477 }
8478
8479 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8480 {
8481 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8482 AssertRC(rc);
8483
8484 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8485 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8486 {
8487 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8488 AssertRC(rc);
8489 }
8490 }
8491
8492 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8493 {
8494 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8495 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8496 }
8497
8498 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8499 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8500 {
8501 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8502 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8503 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8504 AssertRC(rc);
8505 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8506 }
8507
8508 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8509 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8510}
8511
8512
8513/**
8514 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8515 *
8516 * @returns Strict VBox status code (i.e. informational status codes too).
8517 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8518 * without unrestricted guest access and the VMMDev is not presently
8519 * mapped (e.g. EFI32).
8520 *
8521 * @param pVM The cross context VM structure.
8522 * @param pVCpu The cross context virtual CPU structure.
8523 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8524 * out-of-sync. Make sure to update the required fields
8525 * before using them.
8526 *
8527 * @remarks No-long-jump zone!!!
8528 */
8529static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8530{
8531 HMVMX_ASSERT_PREEMPT_SAFE();
8532 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8533 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8534
8535 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8536#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8537 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8538#endif
8539
8540 /*
8541 * RIP is what changes the most often and hence if it's the only bit needing to be
8542 * updated, we shall handle it early for performance reasons.
8543 */
8544 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8545 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8546 {
8547 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8548 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8549 { /* likely */}
8550 else
8551 {
8552 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8553 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8554 }
8555 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8556 }
8557 else if (HMCPU_CF_VALUE(pVCpu))
8558 {
8559 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8560 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8561 { /* likely */}
8562 else
8563 {
8564 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8565 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8566 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8567 return rcStrict;
8568 }
8569 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8570 }
8571
8572 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8573 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8574 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8575 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8576 return rcStrict;
8577}
8578
8579
8580/**
8581 * Does the preparations before executing guest code in VT-x.
8582 *
8583 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8584 * recompiler/IEM. We must be cautious what we do here regarding committing
8585 * guest-state information into the VMCS assuming we assuredly execute the
8586 * guest in VT-x mode.
8587 *
8588 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8589 * the common-state (TRPM/forceflags), we must undo those changes so that the
8590 * recompiler/IEM can (and should) use them when it resumes guest execution.
8591 * Otherwise such operations must be done when we can no longer exit to ring-3.
8592 *
8593 * @returns Strict VBox status code (i.e. informational status codes too).
8594 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8595 * have been disabled.
8596 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8597 * double-fault into the guest.
8598 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8599 * dispatched directly.
8600 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8601 *
8602 * @param pVM The cross context VM structure.
8603 * @param pVCpu The cross context virtual CPU structure.
8604 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8605 * out-of-sync. Make sure to update the required fields
8606 * before using them.
8607 * @param pVmxTransient Pointer to the VMX transient structure.
8608 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8609 * us ignore some of the reasons for returning to
8610 * ring-3, and return VINF_EM_DBG_STEPPED if event
8611 * dispatching took place.
8612 */
8613static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8614{
8615 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8616
8617#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8618 PGMRZDynMapFlushAutoSet(pVCpu);
8619#endif
8620
8621 /* Check force flag actions that might require us to go back to ring-3. */
8622 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8623 if (rcStrict == VINF_SUCCESS)
8624 { /* FFs doesn't get set all the time. */ }
8625 else
8626 return rcStrict;
8627
8628#ifndef IEM_VERIFICATION_MODE_FULL
8629 /*
8630 * Setup the virtualized-APIC accesses.
8631 *
8632 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8633 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8634 *
8635 * This is the reason we do it here and not in hmR0VmxLoadGuestState().
8636 */
8637 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8638 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8639 && PDMHasApic(pVM))
8640 {
8641 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8642 Assert(u64MsrApicBase);
8643 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8644
8645 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8646
8647 /* Unalias any existing mapping. */
8648 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8649 AssertRCReturn(rc, rc);
8650
8651 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8652 LogRel(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8653 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8654 AssertRCReturn(rc, rc);
8655
8656 /* Update the per-VCPU cache of the APIC base MSR. */
8657 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8658 }
8659#endif /* !IEM_VERIFICATION_MODE_FULL */
8660
8661 if (TRPMHasTrap(pVCpu))
8662 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8663 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8664
8665 /*
8666 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8667 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8668 */
8669 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8670 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8671 { /* likely */ }
8672 else
8673 {
8674 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8675 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8676 return rcStrict;
8677 }
8678
8679 /*
8680 * No longjmps to ring-3 from this point on!!!
8681 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8682 * This also disables flushing of the R0-logger instance (if any).
8683 */
8684 VMMRZCallRing3Disable(pVCpu);
8685
8686 /*
8687 * Load the guest state bits.
8688 *
8689 * We cannot perform longjmps while loading the guest state because we do not preserve the
8690 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8691 * CPU migration.
8692 *
8693 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8694 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8695 * Hence, loading of the guest state needs to be done -after- injection of events.
8696 */
8697 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8698 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8699 { /* likely */ }
8700 else
8701 {
8702 VMMRZCallRing3Enable(pVCpu);
8703 return rcStrict;
8704 }
8705
8706 /*
8707 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8708 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8709 *
8710 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8711 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8712 *
8713 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8714 * executing guest code.
8715 */
8716 pVmxTransient->fEFlags = ASMIntDisableFlags();
8717
8718 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8719 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8720 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8721 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8722 {
8723 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8724 {
8725 pVCpu->hm.s.Event.fPending = false;
8726
8727 /*
8728 * We've injected any pending events. This is really the point of no return (to ring-3).
8729 *
8730 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8731 * returns from this function, so don't enable them here.
8732 */
8733 return VINF_SUCCESS;
8734 }
8735
8736 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8737 rcStrict = VINF_EM_RAW_INTERRUPT;
8738 }
8739 else
8740 {
8741 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8742 rcStrict = VINF_EM_RAW_TO_R3;
8743 }
8744
8745 ASMSetFlags(pVmxTransient->fEFlags);
8746 VMMRZCallRing3Enable(pVCpu);
8747
8748 return rcStrict;
8749}
8750
8751
8752/**
8753 * Prepares to run guest code in VT-x and we've committed to doing so. This
8754 * means there is no backing out to ring-3 or anywhere else at this
8755 * point.
8756 *
8757 * @param pVM The cross context VM structure.
8758 * @param pVCpu The cross context virtual CPU structure.
8759 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8760 * out-of-sync. Make sure to update the required fields
8761 * before using them.
8762 * @param pVmxTransient Pointer to the VMX transient structure.
8763 *
8764 * @remarks Called with preemption disabled.
8765 * @remarks No-long-jump zone!!!
8766 */
8767static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8768{
8769 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8770 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8771 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8772
8773 /*
8774 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8775 */
8776 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8777 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8778
8779#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8780 if (!CPUMIsGuestFPUStateActive(pVCpu))
8781 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8782 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8783 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8784#endif
8785
8786 if ( pVCpu->hm.s.fPreloadGuestFpu
8787 && !CPUMIsGuestFPUStateActive(pVCpu))
8788 {
8789 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8790 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8791 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8792 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8793 }
8794
8795 /*
8796 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8797 */
8798 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8799 && pVCpu->hm.s.vmx.cMsrs > 0)
8800 {
8801 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8802 }
8803
8804 /*
8805 * Load the host state bits as we may've been preempted (only happens when
8806 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8807 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8808 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8809 * See @bugref{8432}.
8810 */
8811 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8812 {
8813 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8814 AssertRC(rc);
8815 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8816 }
8817 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8818
8819 /*
8820 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8821 */
8822 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8823 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8824 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8825
8826 /* Store status of the shared guest-host state at the time of VM-entry. */
8827#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8828 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8829 {
8830 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8831 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8832 }
8833 else
8834#endif
8835 {
8836 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8837 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8838 }
8839 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8840
8841 /*
8842 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8843 */
8844 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8845 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8846
8847 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8848 RTCPUID idCurrentCpu = pCpu->idCpu;
8849 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8850 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8851 {
8852 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8853 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8854 }
8855
8856 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8857 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8858 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8859 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8860
8861 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8862
8863 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8864 to start executing. */
8865
8866 /*
8867 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8868 */
8869 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8870 {
8871 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8872 {
8873 bool fMsrUpdated;
8874 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8875 AssertRC(rc2);
8876 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8877
8878 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8879 &fMsrUpdated);
8880 AssertRC(rc2);
8881 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8882
8883 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8884 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8885 }
8886 else
8887 {
8888 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8889 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8890 }
8891 }
8892
8893#ifdef VBOX_STRICT
8894 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8895 hmR0VmxCheckHostEferMsr(pVCpu);
8896 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8897#endif
8898#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8899 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8900 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8901 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8902#endif
8903}
8904
8905
8906/**
8907 * Performs some essential restoration of state after running guest code in
8908 * VT-x.
8909 *
8910 * @param pVM The cross context VM structure.
8911 * @param pVCpu The cross context virtual CPU structure.
8912 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8913 * out-of-sync. Make sure to update the required fields
8914 * before using them.
8915 * @param pVmxTransient Pointer to the VMX transient structure.
8916 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8917 *
8918 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8919 *
8920 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8921 * unconditionally when it is safe to do so.
8922 */
8923static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8924{
8925 NOREF(pVM);
8926
8927 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8928
8929 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8930 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8931 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8932 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8933 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8934 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8935
8936 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8937 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8938
8939 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8940 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8941 Assert(!ASMIntAreEnabled());
8942 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8943
8944#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8945 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8946 {
8947 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8948 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8949 }
8950#endif
8951
8952#if HC_ARCH_BITS == 64
8953 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8954#endif
8955#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8956 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8957 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8958 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8959#else
8960 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8961#endif
8962#ifdef VBOX_STRICT
8963 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8964#endif
8965 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8966 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8967
8968 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8969 uint32_t uExitReason;
8970 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8971 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8972 AssertRC(rc);
8973 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8974 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8975
8976 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8977 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8978 {
8979 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8980 pVmxTransient->fVMEntryFailed));
8981 return;
8982 }
8983
8984 /*
8985 * Update the VM-exit history array here even if the VM-entry failed due to:
8986 * - Invalid guest state.
8987 * - MSR loading.
8988 * - Machine-check event.
8989 *
8990 * In any of the above cases we will still have a "valid" VM-exit reason
8991 * despite @a fVMEntryFailed being false.
8992 *
8993 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8994 */
8995 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8996
8997 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8998 {
8999 /** @todo We can optimize this by only syncing with our force-flags when
9000 * really needed and keeping the VMCS state as it is for most
9001 * VM-exits. */
9002 /* Update the guest interruptibility-state from the VMCS. */
9003 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9004
9005#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9006 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9007 AssertRC(rc);
9008#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9009 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9010 AssertRC(rc);
9011#endif
9012
9013 /*
9014 * Sync the TPR shadow with our APIC state.
9015 */
9016 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9017 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9018 {
9019 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9020 AssertRC(rc);
9021 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9022 }
9023 }
9024}
9025
9026
9027/**
9028 * Runs the guest code using VT-x the normal way.
9029 *
9030 * @returns VBox status code.
9031 * @param pVM The cross context VM structure.
9032 * @param pVCpu The cross context virtual CPU structure.
9033 * @param pCtx Pointer to the guest-CPU context.
9034 *
9035 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9036 */
9037static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9038{
9039 VMXTRANSIENT VmxTransient;
9040 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9041 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9042 uint32_t cLoops = 0;
9043
9044 for (;; cLoops++)
9045 {
9046 Assert(!HMR0SuspendPending());
9047 HMVMX_ASSERT_CPU_SAFE();
9048
9049 /* Preparatory work for running guest code, this may force us to return
9050 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9051 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9052 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9053 if (rcStrict != VINF_SUCCESS)
9054 break;
9055
9056 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9057 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9058 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9059
9060 /* Restore any residual host-state and save any bits shared between host
9061 and guest into the guest-CPU state. Re-enables interrupts! */
9062 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9063
9064 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9065 if (RT_SUCCESS(rcRun))
9066 { /* very likely */ }
9067 else
9068 {
9069 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9070 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9071 return rcRun;
9072 }
9073
9074 /* Profile the VM-exit. */
9075 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9076 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9077 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9078 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9079 HMVMX_START_EXIT_DISPATCH_PROF();
9080
9081 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9082
9083 /* Handle the VM-exit. */
9084#ifdef HMVMX_USE_FUNCTION_TABLE
9085 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9086#else
9087 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9088#endif
9089 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9090 if (rcStrict == VINF_SUCCESS)
9091 {
9092 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9093 continue; /* likely */
9094 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9095 rcStrict = VINF_EM_RAW_INTERRUPT;
9096 }
9097 break;
9098 }
9099
9100 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9101 return rcStrict;
9102}
9103
9104
9105
9106/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9107 * probes.
9108 *
9109 * The following few functions and associated structure contains the bloat
9110 * necessary for providing detailed debug events and dtrace probes as well as
9111 * reliable host side single stepping. This works on the principle of
9112 * "subclassing" the normal execution loop and workers. We replace the loop
9113 * method completely and override selected helpers to add necessary adjustments
9114 * to their core operation.
9115 *
9116 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9117 * any performance for debug and analysis features.
9118 *
9119 * @{
9120 */
9121
9122/**
9123 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9124 * the debug run loop.
9125 */
9126typedef struct VMXRUNDBGSTATE
9127{
9128 /** The RIP we started executing at. This is for detecting that we stepped. */
9129 uint64_t uRipStart;
9130 /** The CS we started executing with. */
9131 uint16_t uCsStart;
9132
9133 /** Whether we've actually modified the 1st execution control field. */
9134 bool fModifiedProcCtls : 1;
9135 /** Whether we've actually modified the 2nd execution control field. */
9136 bool fModifiedProcCtls2 : 1;
9137 /** Whether we've actually modified the exception bitmap. */
9138 bool fModifiedXcptBitmap : 1;
9139
9140 /** We desire the modified the CR0 mask to be cleared. */
9141 bool fClearCr0Mask : 1;
9142 /** We desire the modified the CR4 mask to be cleared. */
9143 bool fClearCr4Mask : 1;
9144 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9145 uint32_t fCpe1Extra;
9146 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9147 uint32_t fCpe1Unwanted;
9148 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9149 uint32_t fCpe2Extra;
9150 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9151 uint32_t bmXcptExtra;
9152 /** The sequence number of the Dtrace provider settings the state was
9153 * configured against. */
9154 uint32_t uDtraceSettingsSeqNo;
9155 /** VM-exits to check (one bit per VM-exit). */
9156 uint32_t bmExitsToCheck[3];
9157
9158 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9159 uint32_t fProcCtlsInitial;
9160 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9161 uint32_t fProcCtls2Initial;
9162 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9163 uint32_t bmXcptInitial;
9164} VMXRUNDBGSTATE;
9165AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9166typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9167
9168
9169/**
9170 * Initializes the VMXRUNDBGSTATE structure.
9171 *
9172 * @param pVCpu The cross context virtual CPU structure of the
9173 * calling EMT.
9174 * @param pCtx The CPU register context to go with @a pVCpu.
9175 * @param pDbgState The structure to initialize.
9176 */
9177DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9178{
9179 pDbgState->uRipStart = pCtx->rip;
9180 pDbgState->uCsStart = pCtx->cs.Sel;
9181
9182 pDbgState->fModifiedProcCtls = false;
9183 pDbgState->fModifiedProcCtls2 = false;
9184 pDbgState->fModifiedXcptBitmap = false;
9185 pDbgState->fClearCr0Mask = false;
9186 pDbgState->fClearCr4Mask = false;
9187 pDbgState->fCpe1Extra = 0;
9188 pDbgState->fCpe1Unwanted = 0;
9189 pDbgState->fCpe2Extra = 0;
9190 pDbgState->bmXcptExtra = 0;
9191 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9192 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9193 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9194}
9195
9196
9197/**
9198 * Updates the VMSC fields with changes requested by @a pDbgState.
9199 *
9200 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9201 * immediately before executing guest code, i.e. when interrupts are disabled.
9202 * We don't check status codes here as we cannot easily assert or return in the
9203 * latter case.
9204 *
9205 * @param pVCpu The cross context virtual CPU structure.
9206 * @param pDbgState The debug state.
9207 */
9208DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9209{
9210 /*
9211 * Ensure desired flags in VMCS control fields are set.
9212 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9213 *
9214 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9215 * there should be no stale data in pCtx at this point.
9216 */
9217 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9218 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9219 {
9220 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9221 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9222 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9223 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9224 pDbgState->fModifiedProcCtls = true;
9225 }
9226
9227 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9228 {
9229 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9230 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9231 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9232 pDbgState->fModifiedProcCtls2 = true;
9233 }
9234
9235 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9236 {
9237 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9238 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9239 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9240 pDbgState->fModifiedXcptBitmap = true;
9241 }
9242
9243 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9244 {
9245 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9246 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9247 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9248 }
9249
9250 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9251 {
9252 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9253 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9254 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9255 }
9256}
9257
9258
9259DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9260{
9261 /*
9262 * Restore VM-exit control settings as we may not reenter this function the
9263 * next time around.
9264 */
9265 /* We reload the initial value, trigger what we can of recalculations the
9266 next time around. From the looks of things, that's all that's required atm. */
9267 if (pDbgState->fModifiedProcCtls)
9268 {
9269 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9270 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9271 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9272 AssertRCReturn(rc2, rc2);
9273 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9274 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9275 }
9276
9277 /* We're currently the only ones messing with this one, so just restore the
9278 cached value and reload the field. */
9279 if ( pDbgState->fModifiedProcCtls2
9280 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9281 {
9282 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9283 AssertRCReturn(rc2, rc2);
9284 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9285 }
9286
9287 /* If we've modified the exception bitmap, we restore it and trigger
9288 reloading and partial recalculation the next time around. */
9289 if (pDbgState->fModifiedXcptBitmap)
9290 {
9291 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9292 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9293 }
9294
9295 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9296 if (pDbgState->fClearCr0Mask)
9297 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9298
9299 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9300 if (pDbgState->fClearCr4Mask)
9301 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9302
9303 return rcStrict;
9304}
9305
9306
9307/**
9308 * Configures VM-exit controls for current DBGF and DTrace settings.
9309 *
9310 * This updates @a pDbgState and the VMCS execution control fields to reflect
9311 * the necessary VM-exits demanded by DBGF and DTrace.
9312 *
9313 * @param pVM The cross context VM structure.
9314 * @param pVCpu The cross context virtual CPU structure.
9315 * @param pCtx Pointer to the guest-CPU context.
9316 * @param pDbgState The debug state.
9317 * @param pVmxTransient Pointer to the VMX transient structure. May update
9318 * fUpdateTscOffsettingAndPreemptTimer.
9319 */
9320static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9321 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9322{
9323 /*
9324 * Take down the dtrace serial number so we can spot changes.
9325 */
9326 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9327 ASMCompilerBarrier();
9328
9329 /*
9330 * We'll rebuild most of the middle block of data members (holding the
9331 * current settings) as we go along here, so start by clearing it all.
9332 */
9333 pDbgState->bmXcptExtra = 0;
9334 pDbgState->fCpe1Extra = 0;
9335 pDbgState->fCpe1Unwanted = 0;
9336 pDbgState->fCpe2Extra = 0;
9337 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9338 pDbgState->bmExitsToCheck[i] = 0;
9339
9340 /*
9341 * Software interrupts (INT XXh) - no idea how to trigger these...
9342 */
9343 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9344 || VBOXVMM_INT_SOFTWARE_ENABLED())
9345 {
9346 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9347 }
9348
9349 /*
9350 * INT3 breakpoints - triggered by #BP exceptions.
9351 */
9352 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9353 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9354
9355 /*
9356 * Exception bitmap and XCPT events+probes.
9357 */
9358 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9359 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9360 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9361
9362 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9363 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9364 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9365 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9366 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9367 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9368 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9369 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9370 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9371 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9372 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9373 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9374 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9375 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9376 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9377 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9378 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9379 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9380
9381 if (pDbgState->bmXcptExtra)
9382 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9383
9384 /*
9385 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9386 *
9387 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9388 * So, when adding/changing/removing please don't forget to update it.
9389 *
9390 * Some of the macros are picking up local variables to save horizontal space,
9391 * (being able to see it in a table is the lesser evil here).
9392 */
9393#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9394 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9395 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9396#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9397 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9398 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9399 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9400 } else do { } while (0)
9401#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9402 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9403 { \
9404 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9405 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9406 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9407 } else do { } while (0)
9408#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9409 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9410 { \
9411 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9412 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9413 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9414 } else do { } while (0)
9415#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9416 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9417 { \
9418 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9419 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9420 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9421 } else do { } while (0)
9422
9423 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9424 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9425 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9426 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9427 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9428
9429 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9430 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9431 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9432 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9433 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9434 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9435 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9436 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9437 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9438 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9439 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9440 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9441 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9442 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9443 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9444 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9445 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9446 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9447 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9448 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9449 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9450 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9451 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9452 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9453 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9454 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9455 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9456 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9457 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9458 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9459 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9460 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9461 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9462 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9463 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9464 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9465
9466 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9467 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9468 {
9469 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9470 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9471 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9472 AssertRC(rc2);
9473
9474#if 0 /** @todo fix me */
9475 pDbgState->fClearCr0Mask = true;
9476 pDbgState->fClearCr4Mask = true;
9477#endif
9478 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9479 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9480 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9481 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9482 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9483 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9484 require clearing here and in the loop if we start using it. */
9485 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9486 }
9487 else
9488 {
9489 if (pDbgState->fClearCr0Mask)
9490 {
9491 pDbgState->fClearCr0Mask = false;
9492 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9493 }
9494 if (pDbgState->fClearCr4Mask)
9495 {
9496 pDbgState->fClearCr4Mask = false;
9497 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9498 }
9499 }
9500 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9501 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9502
9503 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9504 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9505 {
9506 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9507 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9508 }
9509 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9510 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9511
9512 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9513 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9514 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9515 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9516 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9517 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9518 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9519 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9520#if 0 /** @todo too slow, fix handler. */
9521 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9522#endif
9523 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9524
9525 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9526 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9527 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9528 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9529 {
9530 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9531 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9532 }
9533 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9534 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9535 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9536 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9537
9538 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9539 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9540 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9541 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9542 {
9543 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9544 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9545 }
9546 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9547 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9548 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9549 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9550
9551 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9552 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9553 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9554 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9555 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9556 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9557 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9558 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9559 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9560 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9561 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9562 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9563 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9564 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9565 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9566 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9567 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9568 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9569 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9570 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9571 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9572 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9573
9574#undef IS_EITHER_ENABLED
9575#undef SET_ONLY_XBM_IF_EITHER_EN
9576#undef SET_CPE1_XBM_IF_EITHER_EN
9577#undef SET_CPEU_XBM_IF_EITHER_EN
9578#undef SET_CPE2_XBM_IF_EITHER_EN
9579
9580 /*
9581 * Sanitize the control stuff.
9582 */
9583 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9584 if (pDbgState->fCpe2Extra)
9585 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9586 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9587 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9588 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9589 {
9590 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9591 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9592 }
9593
9594 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9595 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9596 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9597 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9598}
9599
9600
9601/**
9602 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9603 * appropriate.
9604 *
9605 * The caller has checked the VM-exit against the
9606 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9607 * already, so we don't have to do that either.
9608 *
9609 * @returns Strict VBox status code (i.e. informational status codes too).
9610 * @param pVM The cross context VM structure.
9611 * @param pVCpu The cross context virtual CPU structure.
9612 * @param pMixedCtx Pointer to the guest-CPU context.
9613 * @param pVmxTransient Pointer to the VMX-transient structure.
9614 * @param uExitReason The VM-exit reason.
9615 *
9616 * @remarks The name of this function is displayed by dtrace, so keep it short
9617 * and to the point. No longer than 33 chars long, please.
9618 */
9619static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9620 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9621{
9622 /*
9623 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9624 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9625 *
9626 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9627 * does. Must add/change/remove both places. Same ordering, please.
9628 *
9629 * Added/removed events must also be reflected in the next section
9630 * where we dispatch dtrace events.
9631 */
9632 bool fDtrace1 = false;
9633 bool fDtrace2 = false;
9634 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9635 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9636 uint32_t uEventArg = 0;
9637#define SET_EXIT(a_EventSubName) \
9638 do { \
9639 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9640 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9641 } while (0)
9642#define SET_BOTH(a_EventSubName) \
9643 do { \
9644 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9645 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9646 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9647 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9648 } while (0)
9649 switch (uExitReason)
9650 {
9651 case VMX_EXIT_MTF:
9652 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9653
9654 case VMX_EXIT_XCPT_OR_NMI:
9655 {
9656 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9657 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9658 {
9659 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9660 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9661 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9662 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9663 {
9664 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9665 {
9666 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9667 uEventArg = pVmxTransient->uExitIntErrorCode;
9668 }
9669 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9670 switch (enmEvent1)
9671 {
9672 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9673 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9674 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9675 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9676 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9677 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9678 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9679 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9680 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9681 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9682 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9683 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9684 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9685 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9686 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9687 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9688 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9689 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9690 default: break;
9691 }
9692 }
9693 else
9694 AssertFailed();
9695 break;
9696
9697 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9698 uEventArg = idxVector;
9699 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9700 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9701 break;
9702 }
9703 break;
9704 }
9705
9706 case VMX_EXIT_TRIPLE_FAULT:
9707 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9708 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9709 break;
9710 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9711 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9712 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9713 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9714 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9715
9716 /* Instruction specific VM-exits: */
9717 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9718 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9719 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9720 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9721 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9722 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9723 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9724 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9725 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9726 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9727 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9728 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9729 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9730 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9731 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9732 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9733 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9734 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9735 case VMX_EXIT_MOV_CRX:
9736 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9737/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9738* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9739 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9740 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9741 SET_BOTH(CRX_READ);
9742 else
9743 SET_BOTH(CRX_WRITE);
9744 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9745 break;
9746 case VMX_EXIT_MOV_DRX:
9747 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9748 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9749 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9750 SET_BOTH(DRX_READ);
9751 else
9752 SET_BOTH(DRX_WRITE);
9753 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9754 break;
9755 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9756 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9757 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9758 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9759 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9760 case VMX_EXIT_XDTR_ACCESS:
9761 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9762 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9763 {
9764 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9765 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9766 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9767 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9768 }
9769 break;
9770
9771 case VMX_EXIT_TR_ACCESS:
9772 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9773 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9774 {
9775 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9776 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9777 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9778 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9779 }
9780 break;
9781
9782 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9783 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9784 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9785 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9786 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9787 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9788 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9789 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9790 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9791 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9792 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9793
9794 /* Events that aren't relevant at this point. */
9795 case VMX_EXIT_EXT_INT:
9796 case VMX_EXIT_INT_WINDOW:
9797 case VMX_EXIT_NMI_WINDOW:
9798 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9799 case VMX_EXIT_PREEMPT_TIMER:
9800 case VMX_EXIT_IO_INSTR:
9801 break;
9802
9803 /* Errors and unexpected events. */
9804 case VMX_EXIT_INIT_SIGNAL:
9805 case VMX_EXIT_SIPI:
9806 case VMX_EXIT_IO_SMI:
9807 case VMX_EXIT_SMI:
9808 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9809 case VMX_EXIT_ERR_MSR_LOAD:
9810 case VMX_EXIT_ERR_MACHINE_CHECK:
9811 break;
9812
9813 default:
9814 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9815 break;
9816 }
9817#undef SET_BOTH
9818#undef SET_EXIT
9819
9820 /*
9821 * Dtrace tracepoints go first. We do them here at once so we don't
9822 * have to copy the guest state saving and stuff a few dozen times.
9823 * Down side is that we've got to repeat the switch, though this time
9824 * we use enmEvent since the probes are a subset of what DBGF does.
9825 */
9826 if (fDtrace1 || fDtrace2)
9827 {
9828 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9829 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9830 switch (enmEvent1)
9831 {
9832 /** @todo consider which extra parameters would be helpful for each probe. */
9833 case DBGFEVENT_END: break;
9834 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9835 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9836 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9837 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9838 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9839 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9840 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9841 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9842 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9843 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9844 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9845 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9846 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9847 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9848 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9849 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9850 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9851 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9852 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9853 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9854 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9855 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9856 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9857 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9858 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9859 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9860 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9861 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9862 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9863 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9864 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9865 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9866 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9867 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9868 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9869 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9870 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9871 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9872 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9873 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9874 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9875 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9876 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9877 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9878 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9879 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9880 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9881 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9882 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9883 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9884 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9885 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9886 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9887 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9888 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9889 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9890 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9891 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9892 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9893 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9894 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9895 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9896 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9897 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9898 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9899 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9900 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9901 }
9902 switch (enmEvent2)
9903 {
9904 /** @todo consider which extra parameters would be helpful for each probe. */
9905 case DBGFEVENT_END: break;
9906 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9907 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9908 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9909 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9910 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9911 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9912 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9913 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9914 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9915 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9916 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9917 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9918 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9919 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9920 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9921 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9922 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9923 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9924 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9925 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9926 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9927 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9928 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9929 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9930 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9931 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9932 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9933 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9934 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9935 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9936 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9937 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9938 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9939 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9940 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9941 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9942 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9943 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9944 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9945 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9946 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9947 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9948 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9949 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9950 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9951 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9952 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9953 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9954 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9955 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9956 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9957 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9958 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9959 }
9960 }
9961
9962 /*
9963 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9964 * the DBGF call will do a full check).
9965 *
9966 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9967 * Note! If we have to events, we prioritize the first, i.e. the instruction
9968 * one, in order to avoid event nesting.
9969 */
9970 if ( enmEvent1 != DBGFEVENT_END
9971 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9972 {
9973 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9974 if (rcStrict != VINF_SUCCESS)
9975 return rcStrict;
9976 }
9977 else if ( enmEvent2 != DBGFEVENT_END
9978 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9979 {
9980 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9981 if (rcStrict != VINF_SUCCESS)
9982 return rcStrict;
9983 }
9984
9985 return VINF_SUCCESS;
9986}
9987
9988
9989/**
9990 * Single-stepping VM-exit filtering.
9991 *
9992 * This is preprocessing the VM-exits and deciding whether we've gotten far
9993 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9994 * handling is performed.
9995 *
9996 * @returns Strict VBox status code (i.e. informational status codes too).
9997 * @param pVM The cross context VM structure.
9998 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9999 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
10000 * out-of-sync. Make sure to update the required
10001 * fields before using them.
10002 * @param pVmxTransient Pointer to the VMX-transient structure.
10003 * @param uExitReason The VM-exit reason.
10004 * @param pDbgState The debug state.
10005 */
10006DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10007 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10008{
10009 /*
10010 * Expensive (saves context) generic dtrace VM-exit probe.
10011 */
10012 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10013 { /* more likely */ }
10014 else
10015 {
10016 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10017 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10018 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10019 }
10020
10021 /*
10022 * Check for host NMI, just to get that out of the way.
10023 */
10024 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10025 { /* normally likely */ }
10026 else
10027 {
10028 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10029 AssertRCReturn(rc2, rc2);
10030 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10031 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10032 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10033 }
10034
10035 /*
10036 * Check for single stepping event if we're stepping.
10037 */
10038 if (pVCpu->hm.s.fSingleInstruction)
10039 {
10040 switch (uExitReason)
10041 {
10042 case VMX_EXIT_MTF:
10043 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10044
10045 /* Various events: */
10046 case VMX_EXIT_XCPT_OR_NMI:
10047 case VMX_EXIT_EXT_INT:
10048 case VMX_EXIT_TRIPLE_FAULT:
10049 case VMX_EXIT_INT_WINDOW:
10050 case VMX_EXIT_NMI_WINDOW:
10051 case VMX_EXIT_TASK_SWITCH:
10052 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10053 case VMX_EXIT_APIC_ACCESS:
10054 case VMX_EXIT_EPT_VIOLATION:
10055 case VMX_EXIT_EPT_MISCONFIG:
10056 case VMX_EXIT_PREEMPT_TIMER:
10057
10058 /* Instruction specific VM-exits: */
10059 case VMX_EXIT_CPUID:
10060 case VMX_EXIT_GETSEC:
10061 case VMX_EXIT_HLT:
10062 case VMX_EXIT_INVD:
10063 case VMX_EXIT_INVLPG:
10064 case VMX_EXIT_RDPMC:
10065 case VMX_EXIT_RDTSC:
10066 case VMX_EXIT_RSM:
10067 case VMX_EXIT_VMCALL:
10068 case VMX_EXIT_VMCLEAR:
10069 case VMX_EXIT_VMLAUNCH:
10070 case VMX_EXIT_VMPTRLD:
10071 case VMX_EXIT_VMPTRST:
10072 case VMX_EXIT_VMREAD:
10073 case VMX_EXIT_VMRESUME:
10074 case VMX_EXIT_VMWRITE:
10075 case VMX_EXIT_VMXOFF:
10076 case VMX_EXIT_VMXON:
10077 case VMX_EXIT_MOV_CRX:
10078 case VMX_EXIT_MOV_DRX:
10079 case VMX_EXIT_IO_INSTR:
10080 case VMX_EXIT_RDMSR:
10081 case VMX_EXIT_WRMSR:
10082 case VMX_EXIT_MWAIT:
10083 case VMX_EXIT_MONITOR:
10084 case VMX_EXIT_PAUSE:
10085 case VMX_EXIT_XDTR_ACCESS:
10086 case VMX_EXIT_TR_ACCESS:
10087 case VMX_EXIT_INVEPT:
10088 case VMX_EXIT_RDTSCP:
10089 case VMX_EXIT_INVVPID:
10090 case VMX_EXIT_WBINVD:
10091 case VMX_EXIT_XSETBV:
10092 case VMX_EXIT_RDRAND:
10093 case VMX_EXIT_INVPCID:
10094 case VMX_EXIT_VMFUNC:
10095 case VMX_EXIT_RDSEED:
10096 case VMX_EXIT_XSAVES:
10097 case VMX_EXIT_XRSTORS:
10098 {
10099 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10100 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10101 AssertRCReturn(rc2, rc2);
10102 if ( pMixedCtx->rip != pDbgState->uRipStart
10103 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10104 return VINF_EM_DBG_STEPPED;
10105 break;
10106 }
10107
10108 /* Errors and unexpected events: */
10109 case VMX_EXIT_INIT_SIGNAL:
10110 case VMX_EXIT_SIPI:
10111 case VMX_EXIT_IO_SMI:
10112 case VMX_EXIT_SMI:
10113 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10114 case VMX_EXIT_ERR_MSR_LOAD:
10115 case VMX_EXIT_ERR_MACHINE_CHECK:
10116 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10117 break;
10118
10119 default:
10120 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10121 break;
10122 }
10123 }
10124
10125 /*
10126 * Check for debugger event breakpoints and dtrace probes.
10127 */
10128 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10129 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10130 {
10131 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10132 if (rcStrict != VINF_SUCCESS)
10133 return rcStrict;
10134 }
10135
10136 /*
10137 * Normal processing.
10138 */
10139#ifdef HMVMX_USE_FUNCTION_TABLE
10140 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10141#else
10142 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10143#endif
10144}
10145
10146
10147/**
10148 * Single steps guest code using VT-x.
10149 *
10150 * @returns Strict VBox status code (i.e. informational status codes too).
10151 * @param pVM The cross context VM structure.
10152 * @param pVCpu The cross context virtual CPU structure.
10153 * @param pCtx Pointer to the guest-CPU context.
10154 *
10155 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10156 */
10157static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10158{
10159 VMXTRANSIENT VmxTransient;
10160 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10161
10162 /* Set HMCPU indicators. */
10163 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10164 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10165 pVCpu->hm.s.fDebugWantRdTscExit = false;
10166 pVCpu->hm.s.fUsingDebugLoop = true;
10167
10168 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10169 VMXRUNDBGSTATE DbgState;
10170 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10171 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10172
10173 /*
10174 * The loop.
10175 */
10176 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10177 for (uint32_t cLoops = 0; ; cLoops++)
10178 {
10179 Assert(!HMR0SuspendPending());
10180 HMVMX_ASSERT_CPU_SAFE();
10181 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10182
10183 /*
10184 * Preparatory work for running guest code, this may force us to return
10185 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10186 */
10187 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10188 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10189 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10190 if (rcStrict != VINF_SUCCESS)
10191 break;
10192
10193 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10194 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10195
10196 /*
10197 * Now we can run the guest code.
10198 */
10199 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10200
10201 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10202
10203 /*
10204 * Restore any residual host-state and save any bits shared between host
10205 * and guest into the guest-CPU state. Re-enables interrupts!
10206 */
10207 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10208
10209 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10210 if (RT_SUCCESS(rcRun))
10211 { /* very likely */ }
10212 else
10213 {
10214 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10215 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10216 return rcRun;
10217 }
10218
10219 /* Profile the VM-exit. */
10220 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10221 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10222 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10223 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10224 HMVMX_START_EXIT_DISPATCH_PROF();
10225
10226 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10227
10228 /*
10229 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10230 */
10231 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10232 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10233 if (rcStrict != VINF_SUCCESS)
10234 break;
10235 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10236 {
10237 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10238 rcStrict = VINF_EM_RAW_INTERRUPT;
10239 break;
10240 }
10241
10242 /*
10243 * Stepping: Did the RIP change, if so, consider it a single step.
10244 * Otherwise, make sure one of the TFs gets set.
10245 */
10246 if (fStepping)
10247 {
10248 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10249 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10250 AssertRCReturn(rc2, rc2);
10251 if ( pCtx->rip != DbgState.uRipStart
10252 || pCtx->cs.Sel != DbgState.uCsStart)
10253 {
10254 rcStrict = VINF_EM_DBG_STEPPED;
10255 break;
10256 }
10257 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10258 }
10259
10260 /*
10261 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10262 */
10263 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10264 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10265 }
10266
10267 /*
10268 * Clear the X86_EFL_TF if necessary.
10269 */
10270 if (pVCpu->hm.s.fClearTrapFlag)
10271 {
10272 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10273 AssertRCReturn(rc2, rc2);
10274 pVCpu->hm.s.fClearTrapFlag = false;
10275 pCtx->eflags.Bits.u1TF = 0;
10276 }
10277 /** @todo there seems to be issues with the resume flag when the monitor trap
10278 * flag is pending without being used. Seen early in bios init when
10279 * accessing APIC page in protected mode. */
10280
10281 /*
10282 * Restore VM-exit control settings as we may not reenter this function the
10283 * next time around.
10284 */
10285 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10286
10287 /* Restore HMCPU indicators. */
10288 pVCpu->hm.s.fUsingDebugLoop = false;
10289 pVCpu->hm.s.fDebugWantRdTscExit = false;
10290 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10291
10292 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10293 return rcStrict;
10294}
10295
10296
10297/** @} */
10298
10299
10300/**
10301 * Checks if any expensive dtrace probes are enabled and we should go to the
10302 * debug loop.
10303 *
10304 * @returns true if we should use debug loop, false if not.
10305 */
10306static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10307{
10308 /* It's probably faster to OR the raw 32-bit counter variables together.
10309 Since the variables are in an array and the probes are next to one
10310 another (more or less), we have good locality. So, better read
10311 eight-nine cache lines ever time and only have one conditional, than
10312 128+ conditionals, right? */
10313 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10314 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10315 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10316 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10317 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10318 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10319 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10320 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10321 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10322 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10323 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10324 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10325 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10326 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10327 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10328 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10329 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10330 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10331 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10332 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10333 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10334 ) != 0
10335 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10336 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10337 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10338 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10339 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10340 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10341 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10342 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10343 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10344 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10345 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10346 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10347 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10348 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10349 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10350 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10351 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10352 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10353 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10354 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10355 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10356 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10357 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10358 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10359 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10360 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10361 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10362 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10363 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10364 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10365 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10366 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10367 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10368 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10369 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10370 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10371 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10372 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10373 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10374 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10375 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10376 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10377 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10378 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10379 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10380 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10381 ) != 0
10382 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10383 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10384 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10385 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10386 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10387 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10388 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10389 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10390 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10391 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10392 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10393 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10394 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10395 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10396 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10397 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10398 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10399 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10400 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10401 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10402 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10403 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10404 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10405 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10406 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10407 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10408 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10409 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10410 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10411 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10412 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10413 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10414 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10415 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10416 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10417 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10418 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10419 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10420 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10421 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10422 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10423 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10424 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10425 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10426 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10427 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10428 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10429 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10430 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10431 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10432 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10433 ) != 0;
10434}
10435
10436
10437/**
10438 * Runs the guest code using VT-x.
10439 *
10440 * @returns Strict VBox status code (i.e. informational status codes too).
10441 * @param pVM The cross context VM structure.
10442 * @param pVCpu The cross context virtual CPU structure.
10443 * @param pCtx Pointer to the guest-CPU context.
10444 */
10445VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10446{
10447 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10448 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10449 HMVMX_ASSERT_PREEMPT_SAFE();
10450
10451 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10452
10453 VBOXSTRICTRC rcStrict;
10454 if ( !pVCpu->hm.s.fUseDebugLoop
10455 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10456 && !DBGFIsStepping(pVCpu)
10457 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10458 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10459 else
10460 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10461
10462 if (rcStrict == VERR_EM_INTERPRETER)
10463 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10464 else if (rcStrict == VINF_EM_RESET)
10465 rcStrict = VINF_EM_TRIPLE_FAULT;
10466
10467 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10468 if (RT_FAILURE(rc2))
10469 {
10470 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10471 rcStrict = rc2;
10472 }
10473 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10474 return rcStrict;
10475}
10476
10477
10478#ifndef HMVMX_USE_FUNCTION_TABLE
10479DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10480{
10481# ifdef DEBUG_ramshankar
10482# define RETURN_EXIT_CALL(a_CallExpr) \
10483 do { \
10484 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10485 VBOXSTRICTRC rcStrict = a_CallExpr; \
10486 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10487 return rcStrict; \
10488 } while (0)
10489# else
10490# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10491# endif
10492 switch (rcReason)
10493 {
10494 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10495 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10496 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10497 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10498 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10499 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10500 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10501 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10502 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10503 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10504 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10505 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10506 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10507 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10508 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10509 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10510 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10511 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10512 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10513 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10514 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10515 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10516 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10517 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10518 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10519 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10520 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10521 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10522 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10523 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10524 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10525 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10526 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10527 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10528
10529 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10530 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10531 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10532 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10533 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10534 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10535 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10536 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10537 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10538
10539 case VMX_EXIT_VMCLEAR:
10540 case VMX_EXIT_VMLAUNCH:
10541 case VMX_EXIT_VMPTRLD:
10542 case VMX_EXIT_VMPTRST:
10543 case VMX_EXIT_VMREAD:
10544 case VMX_EXIT_VMRESUME:
10545 case VMX_EXIT_VMWRITE:
10546 case VMX_EXIT_VMXOFF:
10547 case VMX_EXIT_VMXON:
10548 case VMX_EXIT_INVEPT:
10549 case VMX_EXIT_INVVPID:
10550 case VMX_EXIT_VMFUNC:
10551 case VMX_EXIT_XSAVES:
10552 case VMX_EXIT_XRSTORS:
10553 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10554 case VMX_EXIT_ENCLS:
10555 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10556 case VMX_EXIT_PML_FULL:
10557 default:
10558 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10559 }
10560#undef RETURN_EXIT_CALL
10561}
10562#endif /* !HMVMX_USE_FUNCTION_TABLE */
10563
10564
10565#ifdef VBOX_STRICT
10566/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10567# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10568 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10569
10570# define HMVMX_ASSERT_PREEMPT_CPUID() \
10571 do { \
10572 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10573 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10574 } while (0)
10575
10576# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10577 do { \
10578 AssertPtr(pVCpu); \
10579 AssertPtr(pMixedCtx); \
10580 AssertPtr(pVmxTransient); \
10581 Assert(pVmxTransient->fVMEntryFailed == false); \
10582 Assert(ASMIntAreEnabled()); \
10583 HMVMX_ASSERT_PREEMPT_SAFE(); \
10584 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10585 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10586 HMVMX_ASSERT_PREEMPT_SAFE(); \
10587 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10588 HMVMX_ASSERT_PREEMPT_CPUID(); \
10589 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10590 } while (0)
10591
10592# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10593 do { \
10594 Log4Func(("\n")); \
10595 } while (0)
10596#else /* nonstrict builds: */
10597# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10598 do { \
10599 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10600 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10601 } while (0)
10602# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10603#endif
10604
10605
10606/**
10607 * Advances the guest RIP by the specified number of bytes.
10608 *
10609 * @param pVCpu The cross context virtual CPU structure.
10610 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10611 * out-of-sync. Make sure to update the required fields
10612 * before using them.
10613 * @param cbInstr Number of bytes to advance the RIP by.
10614 *
10615 * @remarks No-long-jump zone!!!
10616 */
10617DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10618{
10619 /* Advance the RIP. */
10620 pMixedCtx->rip += cbInstr;
10621 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10622
10623 /* Update interrupt inhibition. */
10624 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10625 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10626 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10627}
10628
10629
10630/**
10631 * Advances the guest RIP after reading it from the VMCS.
10632 *
10633 * @returns VBox status code, no informational status codes.
10634 * @param pVCpu The cross context virtual CPU structure.
10635 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10636 * out-of-sync. Make sure to update the required fields
10637 * before using them.
10638 * @param pVmxTransient Pointer to the VMX transient structure.
10639 *
10640 * @remarks No-long-jump zone!!!
10641 */
10642static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10643{
10644 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10645 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10646 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10647 AssertRCReturn(rc, rc);
10648
10649 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10650
10651 /*
10652 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10653 * pending debug exception field as it takes care of priority of events.
10654 *
10655 * See Intel spec. 32.2.1 "Debug Exceptions".
10656 */
10657 if ( !pVCpu->hm.s.fSingleInstruction
10658 && pMixedCtx->eflags.Bits.u1TF)
10659 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10660
10661 return VINF_SUCCESS;
10662}
10663
10664
10665/**
10666 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10667 * and update error record fields accordingly.
10668 *
10669 * @return VMX_IGS_* return codes.
10670 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10671 * wrong with the guest state.
10672 *
10673 * @param pVM The cross context VM structure.
10674 * @param pVCpu The cross context virtual CPU structure.
10675 * @param pCtx Pointer to the guest-CPU state.
10676 *
10677 * @remarks This function assumes our cache of the VMCS controls
10678 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10679 */
10680static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10681{
10682#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10683#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10684 uError = (err); \
10685 break; \
10686 } else do { } while (0)
10687
10688 int rc;
10689 uint32_t uError = VMX_IGS_ERROR;
10690 uint32_t u32Val;
10691 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10692
10693 do
10694 {
10695 /*
10696 * CR0.
10697 */
10698 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10699 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10700 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10701 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10702 if (fUnrestrictedGuest)
10703 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10704
10705 uint32_t u32GuestCR0;
10706 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10707 AssertRCBreak(rc);
10708 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10709 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10710 if ( !fUnrestrictedGuest
10711 && (u32GuestCR0 & X86_CR0_PG)
10712 && !(u32GuestCR0 & X86_CR0_PE))
10713 {
10714 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10715 }
10716
10717 /*
10718 * CR4.
10719 */
10720 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10721 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10722
10723 uint32_t u32GuestCR4;
10724 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10725 AssertRCBreak(rc);
10726 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10727 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10728
10729 /*
10730 * IA32_DEBUGCTL MSR.
10731 */
10732 uint64_t u64Val;
10733 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10734 AssertRCBreak(rc);
10735 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10736 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10737 {
10738 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10739 }
10740 uint64_t u64DebugCtlMsr = u64Val;
10741
10742#ifdef VBOX_STRICT
10743 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10744 AssertRCBreak(rc);
10745 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10746#endif
10747 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10748
10749 /*
10750 * RIP and RFLAGS.
10751 */
10752 uint32_t u32Eflags;
10753#if HC_ARCH_BITS == 64
10754 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10755 AssertRCBreak(rc);
10756 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10757 if ( !fLongModeGuest
10758 || !pCtx->cs.Attr.n.u1Long)
10759 {
10760 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10761 }
10762 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10763 * must be identical if the "IA-32e mode guest" VM-entry
10764 * control is 1 and CS.L is 1. No check applies if the
10765 * CPU supports 64 linear-address bits. */
10766
10767 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10768 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10769 AssertRCBreak(rc);
10770 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10771 VMX_IGS_RFLAGS_RESERVED);
10772 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10773 u32Eflags = u64Val;
10774#else
10775 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10776 AssertRCBreak(rc);
10777 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10778 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10779#endif
10780
10781 if ( fLongModeGuest
10782 || ( fUnrestrictedGuest
10783 && !(u32GuestCR0 & X86_CR0_PE)))
10784 {
10785 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10786 }
10787
10788 uint32_t u32EntryInfo;
10789 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10790 AssertRCBreak(rc);
10791 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10792 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10793 {
10794 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10795 }
10796
10797 /*
10798 * 64-bit checks.
10799 */
10800#if HC_ARCH_BITS == 64
10801 if (fLongModeGuest)
10802 {
10803 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10804 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10805 }
10806
10807 if ( !fLongModeGuest
10808 && (u32GuestCR4 & X86_CR4_PCIDE))
10809 {
10810 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10811 }
10812
10813 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10814 * 51:32 beyond the processor's physical-address width are 0. */
10815
10816 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10817 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10818 {
10819 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10820 }
10821
10822 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10823 AssertRCBreak(rc);
10824 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10825
10826 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10827 AssertRCBreak(rc);
10828 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10829#endif
10830
10831 /*
10832 * PERF_GLOBAL MSR.
10833 */
10834 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10835 {
10836 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10837 AssertRCBreak(rc);
10838 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10839 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10840 }
10841
10842 /*
10843 * PAT MSR.
10844 */
10845 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10846 {
10847 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10848 AssertRCBreak(rc);
10849 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10850 for (unsigned i = 0; i < 8; i++)
10851 {
10852 uint8_t u8Val = (u64Val & 0xff);
10853 if ( u8Val != 0 /* UC */
10854 && u8Val != 1 /* WC */
10855 && u8Val != 4 /* WT */
10856 && u8Val != 5 /* WP */
10857 && u8Val != 6 /* WB */
10858 && u8Val != 7 /* UC- */)
10859 {
10860 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10861 }
10862 u64Val >>= 8;
10863 }
10864 }
10865
10866 /*
10867 * EFER MSR.
10868 */
10869 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10870 {
10871 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10872 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10873 AssertRCBreak(rc);
10874 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10875 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10876 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10877 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10878 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10879 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10880 || !(u32GuestCR0 & X86_CR0_PG)
10881 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10882 VMX_IGS_EFER_LMA_LME_MISMATCH);
10883 }
10884
10885 /*
10886 * Segment registers.
10887 */
10888 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10889 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10890 if (!(u32Eflags & X86_EFL_VM))
10891 {
10892 /* CS */
10893 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10894 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10895 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10896 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10897 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10898 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10899 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10900 /* CS cannot be loaded with NULL in protected mode. */
10901 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10902 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10903 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10904 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10905 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10906 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10907 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10908 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10909 else
10910 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10911
10912 /* SS */
10913 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10914 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10915 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10916 if ( !(pCtx->cr0 & X86_CR0_PE)
10917 || pCtx->cs.Attr.n.u4Type == 3)
10918 {
10919 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10920 }
10921 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10922 {
10923 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10924 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10925 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10926 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10927 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10928 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10929 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10930 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10931 }
10932
10933 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10934 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10935 {
10936 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10937 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10938 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10939 || pCtx->ds.Attr.n.u4Type > 11
10940 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10941 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10942 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10943 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10944 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10945 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10946 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10947 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10948 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10949 }
10950 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10951 {
10952 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10953 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10954 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10955 || pCtx->es.Attr.n.u4Type > 11
10956 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10957 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10958 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10959 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10960 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10961 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10962 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10963 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10964 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10965 }
10966 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10967 {
10968 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10969 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10970 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10971 || pCtx->fs.Attr.n.u4Type > 11
10972 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10973 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10974 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10975 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10976 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10977 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10978 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10979 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10980 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10981 }
10982 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10983 {
10984 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10985 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10986 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10987 || pCtx->gs.Attr.n.u4Type > 11
10988 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10989 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10990 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10991 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10992 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10993 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10994 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10995 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10996 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10997 }
10998 /* 64-bit capable CPUs. */
10999#if HC_ARCH_BITS == 64
11000 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11001 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11002 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11003 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11004 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11005 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11006 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11007 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11008 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11009 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11010 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11011#endif
11012 }
11013 else
11014 {
11015 /* V86 mode checks. */
11016 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11017 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11018 {
11019 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11020 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11021 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11022 }
11023 else
11024 {
11025 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11026 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11027 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11028 }
11029
11030 /* CS */
11031 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11032 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11033 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11034 /* SS */
11035 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11036 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11037 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11038 /* DS */
11039 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11040 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11041 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11042 /* ES */
11043 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11044 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11045 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11046 /* FS */
11047 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11048 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11049 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11050 /* GS */
11051 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11052 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11053 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11054 /* 64-bit capable CPUs. */
11055#if HC_ARCH_BITS == 64
11056 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11057 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11058 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11059 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11060 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11061 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11062 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11063 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11064 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11065 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11066 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11067#endif
11068 }
11069
11070 /*
11071 * TR.
11072 */
11073 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11074 /* 64-bit capable CPUs. */
11075#if HC_ARCH_BITS == 64
11076 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11077#endif
11078 if (fLongModeGuest)
11079 {
11080 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11081 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11082 }
11083 else
11084 {
11085 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11086 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11087 VMX_IGS_TR_ATTR_TYPE_INVALID);
11088 }
11089 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11090 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11091 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11092 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11093 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11094 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11095 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11096 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11097
11098 /*
11099 * GDTR and IDTR.
11100 */
11101#if HC_ARCH_BITS == 64
11102 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11103 AssertRCBreak(rc);
11104 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11105
11106 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11107 AssertRCBreak(rc);
11108 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11109#endif
11110
11111 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11112 AssertRCBreak(rc);
11113 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11114
11115 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11116 AssertRCBreak(rc);
11117 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11118
11119 /*
11120 * Guest Non-Register State.
11121 */
11122 /* Activity State. */
11123 uint32_t u32ActivityState;
11124 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11125 AssertRCBreak(rc);
11126 HMVMX_CHECK_BREAK( !u32ActivityState
11127 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11128 VMX_IGS_ACTIVITY_STATE_INVALID);
11129 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11130 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11131 uint32_t u32IntrState;
11132 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11133 AssertRCBreak(rc);
11134 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11135 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11136 {
11137 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11138 }
11139
11140 /** @todo Activity state and injecting interrupts. Left as a todo since we
11141 * currently don't use activity states but ACTIVE. */
11142
11143 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11144 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11145
11146 /* Guest interruptibility-state. */
11147 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11148 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11149 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11150 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11151 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11152 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11153 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11154 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11155 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11156 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11157 {
11158 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11159 {
11160 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11161 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11162 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11163 }
11164 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11165 {
11166 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11167 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11168 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11169 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11170 }
11171 }
11172 /** @todo Assumes the processor is not in SMM. */
11173 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11174 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11175 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11176 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11177 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11178 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11179 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11180 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11181 {
11182 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11183 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11184 }
11185
11186 /* Pending debug exceptions. */
11187#if HC_ARCH_BITS == 64
11188 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11189 AssertRCBreak(rc);
11190 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11191 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11192 u32Val = u64Val; /* For pending debug exceptions checks below. */
11193#else
11194 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11195 AssertRCBreak(rc);
11196 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11197 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11198#endif
11199
11200 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11201 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11202 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11203 {
11204 if ( (u32Eflags & X86_EFL_TF)
11205 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11206 {
11207 /* Bit 14 is PendingDebug.BS. */
11208 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11209 }
11210 if ( !(u32Eflags & X86_EFL_TF)
11211 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11212 {
11213 /* Bit 14 is PendingDebug.BS. */
11214 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11215 }
11216 }
11217
11218 /* VMCS link pointer. */
11219 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11220 AssertRCBreak(rc);
11221 if (u64Val != UINT64_C(0xffffffffffffffff))
11222 {
11223 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11224 /** @todo Bits beyond the processor's physical-address width MBZ. */
11225 /** @todo 32-bit located in memory referenced by value of this field (as a
11226 * physical address) must contain the processor's VMCS revision ID. */
11227 /** @todo SMM checks. */
11228 }
11229
11230 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11231 * not using Nested Paging? */
11232 if ( pVM->hm.s.fNestedPaging
11233 && !fLongModeGuest
11234 && CPUMIsGuestInPAEModeEx(pCtx))
11235 {
11236 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11237 AssertRCBreak(rc);
11238 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11239
11240 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11241 AssertRCBreak(rc);
11242 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11243
11244 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11245 AssertRCBreak(rc);
11246 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11247
11248 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11249 AssertRCBreak(rc);
11250 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11251 }
11252
11253 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11254 if (uError == VMX_IGS_ERROR)
11255 uError = VMX_IGS_REASON_NOT_FOUND;
11256 } while (0);
11257
11258 pVCpu->hm.s.u32HMError = uError;
11259 return uError;
11260
11261#undef HMVMX_ERROR_BREAK
11262#undef HMVMX_CHECK_BREAK
11263}
11264
11265/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11266/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11267/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11268
11269/** @name VM-exit handlers.
11270 * @{
11271 */
11272
11273/**
11274 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11275 */
11276HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11277{
11278 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11279 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11280 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11281 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11282 return VINF_SUCCESS;
11283 return VINF_EM_RAW_INTERRUPT;
11284}
11285
11286
11287/**
11288 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11289 */
11290HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11291{
11292 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11293 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11294
11295 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11296 AssertRCReturn(rc, rc);
11297
11298 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11299 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11300 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11301 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11302
11303 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11304 {
11305 /*
11306 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11307 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11308 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11309 *
11310 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11311 */
11312 VMXDispatchHostNmi();
11313 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11314 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11315 return VINF_SUCCESS;
11316 }
11317
11318 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11319 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11320 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11321 { /* likely */ }
11322 else
11323 {
11324 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11325 rcStrictRc1 = VINF_SUCCESS;
11326 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11327 return rcStrictRc1;
11328 }
11329
11330 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11331 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11332 switch (uIntType)
11333 {
11334 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11335 Assert(uVector == X86_XCPT_DB);
11336 /* no break */
11337 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11338 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11339 /* no break */
11340 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11341 {
11342 /*
11343 * If there's any exception caused as a result of event injection, go back to
11344 * the interpreter. The page-fault case is complicated and we manually handle
11345 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11346 * handled in hmR0VmxCheckExitDueToEventDelivery.
11347 */
11348 if (!pVCpu->hm.s.Event.fPending)
11349 { /* likely */ }
11350 else if ( uVector != X86_XCPT_PF
11351 && uVector != X86_XCPT_AC)
11352 {
11353 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11354 rc = VERR_EM_INTERPRETER;
11355 break;
11356 }
11357
11358 switch (uVector)
11359 {
11360 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11361 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11362 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11363 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11364 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11365 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11366 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11367
11368 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11369 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11370 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11371 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11372 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11373 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11374 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11375 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11376 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11377 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11378 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11379 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11380 default:
11381 {
11382 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11383 AssertRCReturn(rc, rc);
11384
11385 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11386 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11387 {
11388 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11389 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11390 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11391
11392 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11393 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11394 AssertRCReturn(rc, rc);
11395 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11396 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11397 0 /* GCPtrFaultAddress */);
11398 AssertRCReturn(rc, rc);
11399 }
11400 else
11401 {
11402 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11403 pVCpu->hm.s.u32HMError = uVector;
11404 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11405 }
11406 break;
11407 }
11408 }
11409 break;
11410 }
11411
11412 default:
11413 {
11414 pVCpu->hm.s.u32HMError = uExitIntInfo;
11415 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11416 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11417 break;
11418 }
11419 }
11420 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11421 return rc;
11422}
11423
11424
11425/**
11426 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11427 */
11428HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11429{
11430 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11431
11432 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11433 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11434
11435 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11436 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11437 return VINF_SUCCESS;
11438}
11439
11440
11441/**
11442 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11443 */
11444HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11445{
11446 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11447 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11448 {
11449 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11450 HMVMX_RETURN_UNEXPECTED_EXIT();
11451 }
11452
11453 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11454
11455 /*
11456 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11457 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11458 */
11459 uint32_t uIntrState = 0;
11460 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11461 AssertRCReturn(rc, rc);
11462
11463 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11464 if ( fBlockSti
11465 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11466 {
11467 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11468 }
11469
11470 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11471 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11472
11473 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11474 return VINF_SUCCESS;
11475}
11476
11477
11478/**
11479 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11480 */
11481HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11482{
11483 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11484 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11485 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11486}
11487
11488
11489/**
11490 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11491 */
11492HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11493{
11494 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11495 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11496 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11497}
11498
11499
11500/**
11501 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11502 */
11503HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11504{
11505 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11506 PVM pVM = pVCpu->CTX_SUFF(pVM);
11507 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11508 if (RT_LIKELY(rc == VINF_SUCCESS))
11509 {
11510 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11511 Assert(pVmxTransient->cbInstr == 2);
11512 }
11513 else
11514 {
11515 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11516 rc = VERR_EM_INTERPRETER;
11517 }
11518 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11519 return rc;
11520}
11521
11522
11523/**
11524 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11525 */
11526HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11527{
11528 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11529 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11530 AssertRCReturn(rc, rc);
11531
11532 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11533 return VINF_EM_RAW_EMULATE_INSTR;
11534
11535 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11536 HMVMX_RETURN_UNEXPECTED_EXIT();
11537}
11538
11539
11540/**
11541 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11542 */
11543HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11544{
11545 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11546 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11547 AssertRCReturn(rc, rc);
11548
11549 PVM pVM = pVCpu->CTX_SUFF(pVM);
11550 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11551 if (RT_LIKELY(rc == VINF_SUCCESS))
11552 {
11553 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11554 Assert(pVmxTransient->cbInstr == 2);
11555 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11556 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11557 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11558 }
11559 else
11560 rc = VERR_EM_INTERPRETER;
11561 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11562 return rc;
11563}
11564
11565
11566/**
11567 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11568 */
11569HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11570{
11571 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11572 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11573 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11574 AssertRCReturn(rc, rc);
11575
11576 PVM pVM = pVCpu->CTX_SUFF(pVM);
11577 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11578 if (RT_SUCCESS(rc))
11579 {
11580 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11581 Assert(pVmxTransient->cbInstr == 3);
11582 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11583 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11584 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11585 }
11586 else
11587 {
11588 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11589 rc = VERR_EM_INTERPRETER;
11590 }
11591 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11592 return rc;
11593}
11594
11595
11596/**
11597 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11598 */
11599HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11600{
11601 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11602 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11603 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11604 AssertRCReturn(rc, rc);
11605
11606 PVM pVM = pVCpu->CTX_SUFF(pVM);
11607 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11608 if (RT_LIKELY(rc == VINF_SUCCESS))
11609 {
11610 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11611 Assert(pVmxTransient->cbInstr == 2);
11612 }
11613 else
11614 {
11615 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11616 rc = VERR_EM_INTERPRETER;
11617 }
11618 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11619 return rc;
11620}
11621
11622
11623/**
11624 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11625 */
11626HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11627{
11628 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11629 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11630
11631 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11632 if (pVCpu->hm.s.fHypercallsEnabled)
11633 {
11634#if 0
11635 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11636#else
11637 /* Aggressive state sync. for now. */
11638 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11639 rc |= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11640 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11641 AssertRCReturn(rc, rc);
11642#endif
11643
11644 /* Perform the hypercall. */
11645 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11646 if (rcStrict == VINF_SUCCESS)
11647 {
11648 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11649 AssertRCReturn(rc, rc);
11650 }
11651 else
11652 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11653 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11654 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11655
11656 /* If the hypercall changes anything other than guest's general-purpose registers,
11657 we would need to reload the guest changed bits here before VM-entry. */
11658 }
11659 else
11660 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11661
11662 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11663 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11664 {
11665 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11666 rcStrict = VINF_SUCCESS;
11667 }
11668
11669 return rcStrict;
11670}
11671
11672
11673/**
11674 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11675 */
11676HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11677{
11678 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11679 PVM pVM = pVCpu->CTX_SUFF(pVM);
11680 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11681
11682 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11683 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11684 AssertRCReturn(rc, rc);
11685
11686 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11687 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11688 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11689 else
11690 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11691 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11692 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11693 return rcStrict;
11694}
11695
11696
11697/**
11698 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11699 */
11700HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11701{
11702 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11703 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11704 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11705 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11706 AssertRCReturn(rc, rc);
11707
11708 PVM pVM = pVCpu->CTX_SUFF(pVM);
11709 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11710 if (RT_LIKELY(rc == VINF_SUCCESS))
11711 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11712 else
11713 {
11714 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11715 rc = VERR_EM_INTERPRETER;
11716 }
11717 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11718 return rc;
11719}
11720
11721
11722/**
11723 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11724 */
11725HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11726{
11727 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11728 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11729 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11730 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11731 AssertRCReturn(rc, rc);
11732
11733 PVM pVM = pVCpu->CTX_SUFF(pVM);
11734 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11735 rc = VBOXSTRICTRC_VAL(rc2);
11736 if (RT_LIKELY( rc == VINF_SUCCESS
11737 || rc == VINF_EM_HALT))
11738 {
11739 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11740 AssertRCReturn(rc3, rc3);
11741
11742 if ( rc == VINF_EM_HALT
11743 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11744 {
11745 rc = VINF_SUCCESS;
11746 }
11747 }
11748 else
11749 {
11750 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11751 rc = VERR_EM_INTERPRETER;
11752 }
11753 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11754 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11755 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11756 return rc;
11757}
11758
11759
11760/**
11761 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11762 */
11763HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11764{
11765 /*
11766 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11767 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11768 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11769 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11770 */
11771 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11772 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11773 HMVMX_RETURN_UNEXPECTED_EXIT();
11774}
11775
11776
11777/**
11778 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11779 */
11780HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11781{
11782 /*
11783 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11784 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11785 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11786 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11787 */
11788 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11789 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11790 HMVMX_RETURN_UNEXPECTED_EXIT();
11791}
11792
11793
11794/**
11795 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11796 */
11797HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11798{
11799 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11800 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11801 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11802 HMVMX_RETURN_UNEXPECTED_EXIT();
11803}
11804
11805
11806/**
11807 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11808 */
11809HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11810{
11811 /*
11812 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11813 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11814 * See Intel spec. 25.3 "Other Causes of VM-exits".
11815 */
11816 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11817 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11818 HMVMX_RETURN_UNEXPECTED_EXIT();
11819}
11820
11821
11822/**
11823 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11824 * VM-exit.
11825 */
11826HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11827{
11828 /*
11829 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11830 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11831 *
11832 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11833 * See Intel spec. "23.8 Restrictions on VMX operation".
11834 */
11835 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11836 return VINF_SUCCESS;
11837}
11838
11839
11840/**
11841 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11842 * VM-exit.
11843 */
11844HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11845{
11846 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11847 return VINF_EM_RESET;
11848}
11849
11850
11851/**
11852 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11853 */
11854HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11855{
11856 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11857 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11858
11859 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11860 AssertRCReturn(rc, rc);
11861
11862 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11863 rc = VINF_SUCCESS;
11864 else
11865 rc = VINF_EM_HALT;
11866
11867 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11868 if (rc != VINF_SUCCESS)
11869 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11870 return rc;
11871}
11872
11873
11874/**
11875 * VM-exit handler for instructions that result in a \#UD exception delivered to
11876 * the guest.
11877 */
11878HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11879{
11880 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11881 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11882 return VINF_SUCCESS;
11883}
11884
11885
11886/**
11887 * VM-exit handler for expiry of the VMX preemption timer.
11888 */
11889HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11890{
11891 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11892
11893 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11894 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11895
11896 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11897 PVM pVM = pVCpu->CTX_SUFF(pVM);
11898 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11899 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11900 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11901}
11902
11903
11904/**
11905 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11906 */
11907HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11908{
11909 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11910
11911 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11912 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11913 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11914 AssertRCReturn(rc, rc);
11915
11916 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11917 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11918
11919 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11920
11921 return rcStrict;
11922}
11923
11924
11925/**
11926 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11927 */
11928HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11929{
11930 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11931
11932 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11933 /** @todo implement EMInterpretInvpcid() */
11934 return VERR_EM_INTERPRETER;
11935}
11936
11937
11938/**
11939 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11940 * Error VM-exit.
11941 */
11942HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11943{
11944 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11945 AssertRCReturn(rc, rc);
11946
11947 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11948 AssertRCReturn(rc, rc);
11949
11950 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11951 NOREF(uInvalidReason);
11952
11953#ifdef VBOX_STRICT
11954 uint32_t uIntrState;
11955 RTHCUINTREG uHCReg;
11956 uint64_t u64Val;
11957 uint32_t u32Val;
11958
11959 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11960 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11961 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11962 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11963 AssertRCReturn(rc, rc);
11964
11965 Log4(("uInvalidReason %u\n", uInvalidReason));
11966 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11967 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11968 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11969 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11970
11971 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11972 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11973 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11974 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11975 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11976 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11977 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11978 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11979 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11980 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11981 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11982 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11983#else
11984 NOREF(pVmxTransient);
11985#endif
11986
11987 hmDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11988 return VERR_VMX_INVALID_GUEST_STATE;
11989}
11990
11991
11992/**
11993 * VM-exit handler for VM-entry failure due to an MSR-load
11994 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11995 */
11996HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11997{
11998 NOREF(pVmxTransient);
11999 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12000 HMVMX_RETURN_UNEXPECTED_EXIT();
12001}
12002
12003
12004/**
12005 * VM-exit handler for VM-entry failure due to a machine-check event
12006 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12007 */
12008HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12009{
12010 NOREF(pVmxTransient);
12011 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12012 HMVMX_RETURN_UNEXPECTED_EXIT();
12013}
12014
12015
12016/**
12017 * VM-exit handler for all undefined reasons. Should never ever happen.. in
12018 * theory.
12019 */
12020HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12021{
12022 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12023 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12024 return VERR_VMX_UNDEFINED_EXIT_CODE;
12025}
12026
12027
12028/**
12029 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12030 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12031 * Conditional VM-exit.
12032 */
12033HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12034{
12035 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12036
12037 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12038 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12039 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12040 return VERR_EM_INTERPRETER;
12041 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12042 HMVMX_RETURN_UNEXPECTED_EXIT();
12043}
12044
12045
12046/**
12047 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12048 */
12049HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12050{
12051 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12052
12053 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12054 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12055 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12056 return VERR_EM_INTERPRETER;
12057 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12058 HMVMX_RETURN_UNEXPECTED_EXIT();
12059}
12060
12061
12062/**
12063 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12064 */
12065HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12066{
12067 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12068
12069 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12070 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12071 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12072 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12073 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12074 {
12075 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12076 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12077 }
12078 AssertRCReturn(rc, rc);
12079 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12080
12081#ifdef VBOX_STRICT
12082 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12083 {
12084 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12085 && pMixedCtx->ecx != MSR_K6_EFER)
12086 {
12087 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12088 pMixedCtx->ecx));
12089 HMVMX_RETURN_UNEXPECTED_EXIT();
12090 }
12091 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12092 {
12093 VMXMSREXITREAD enmRead;
12094 VMXMSREXITWRITE enmWrite;
12095 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12096 AssertRCReturn(rc2, rc2);
12097 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12098 {
12099 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12100 HMVMX_RETURN_UNEXPECTED_EXIT();
12101 }
12102 }
12103 }
12104#endif
12105
12106 PVM pVM = pVCpu->CTX_SUFF(pVM);
12107 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12108 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12109 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12110 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12111 if (RT_SUCCESS(rc))
12112 {
12113 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12114 Assert(pVmxTransient->cbInstr == 2);
12115 }
12116 return rc;
12117}
12118
12119
12120/**
12121 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12122 */
12123HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12124{
12125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12126 PVM pVM = pVCpu->CTX_SUFF(pVM);
12127 int rc = VINF_SUCCESS;
12128
12129 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12130 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12131 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12132 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12133 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12134 {
12135 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12136 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12137 }
12138 AssertRCReturn(rc, rc);
12139 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12140
12141 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12142 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12143 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12144
12145 if (RT_SUCCESS(rc))
12146 {
12147 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12148
12149 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12150 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12151 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12152 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12153 {
12154 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12155 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12156 EMInterpretWrmsr() changes it. */
12157 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12158 }
12159 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12160 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12161 else if (pMixedCtx->ecx == MSR_K6_EFER)
12162 {
12163 /*
12164 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12165 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12166 * the other bits as well, SCE and NXE. See @bugref{7368}.
12167 */
12168 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12169 }
12170
12171 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12172 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12173 {
12174 switch (pMixedCtx->ecx)
12175 {
12176 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12177 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12178 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12179 case MSR_K8_FS_BASE: /* no break */
12180 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12181 case MSR_K6_EFER: /* already handled above */ break;
12182 default:
12183 {
12184 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12185 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12186 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12187 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12188 break;
12189 }
12190 }
12191 }
12192#ifdef VBOX_STRICT
12193 else
12194 {
12195 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12196 switch (pMixedCtx->ecx)
12197 {
12198 case MSR_IA32_SYSENTER_CS:
12199 case MSR_IA32_SYSENTER_EIP:
12200 case MSR_IA32_SYSENTER_ESP:
12201 case MSR_K8_FS_BASE:
12202 case MSR_K8_GS_BASE:
12203 {
12204 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12205 HMVMX_RETURN_UNEXPECTED_EXIT();
12206 }
12207
12208 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12209 default:
12210 {
12211 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12212 {
12213 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12214 if (pMixedCtx->ecx != MSR_K6_EFER)
12215 {
12216 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12217 pMixedCtx->ecx));
12218 HMVMX_RETURN_UNEXPECTED_EXIT();
12219 }
12220 }
12221
12222 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12223 {
12224 VMXMSREXITREAD enmRead;
12225 VMXMSREXITWRITE enmWrite;
12226 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12227 AssertRCReturn(rc2, rc2);
12228 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12229 {
12230 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12231 HMVMX_RETURN_UNEXPECTED_EXIT();
12232 }
12233 }
12234 break;
12235 }
12236 }
12237 }
12238#endif /* VBOX_STRICT */
12239 }
12240 return rc;
12241}
12242
12243
12244/**
12245 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12246 */
12247HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12248{
12249 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12250
12251 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12252 return VINF_EM_RAW_INTERRUPT;
12253}
12254
12255
12256/**
12257 * VM-exit handler for when the TPR value is lowered below the specified
12258 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12259 */
12260HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12261{
12262 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12263 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12264
12265 /*
12266 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12267 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12268 */
12269 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12270 return VINF_SUCCESS;
12271}
12272
12273
12274/**
12275 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12276 * VM-exit.
12277 *
12278 * @retval VINF_SUCCESS when guest execution can continue.
12279 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12280 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12281 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12282 * interpreter.
12283 */
12284HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12285{
12286 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12287 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12288 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12289 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12290 AssertRCReturn(rc, rc);
12291
12292 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12293 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12294 PVM pVM = pVCpu->CTX_SUFF(pVM);
12295 VBOXSTRICTRC rcStrict;
12296 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12297 switch (uAccessType)
12298 {
12299 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12300 {
12301 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12302 AssertRCReturn(rc, rc);
12303
12304 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12305 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12306 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12307 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12308 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12309 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12310 {
12311 case 0: /* CR0 */
12312 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12313 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12314 break;
12315 case 2: /* CR2 */
12316 /* Nothing to do here, CR2 it's not part of the VMCS. */
12317 break;
12318 case 3: /* CR3 */
12319 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12320 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12321 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12322 break;
12323 case 4: /* CR4 */
12324 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12325 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12326 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12327 break;
12328 case 8: /* CR8 */
12329 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12330 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12331 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12332 break;
12333 default:
12334 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12335 break;
12336 }
12337
12338 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12339 break;
12340 }
12341
12342 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12343 {
12344 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12345 AssertRCReturn(rc, rc);
12346
12347 Assert( !pVM->hm.s.fNestedPaging
12348 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12349 || pVCpu->hm.s.fUsingDebugLoop
12350 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12351
12352 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12353 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12354 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12355
12356 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12357 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12358 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12359 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12360 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12361 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12362 VBOXSTRICTRC_VAL(rcStrict)));
12363 break;
12364 }
12365
12366 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12367 {
12368 AssertRCReturn(rc, rc);
12369 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12370 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12371 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12372 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12373 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12374 break;
12375 }
12376
12377 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12378 {
12379 AssertRCReturn(rc, rc);
12380 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12381 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12382 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12383 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12384 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12385 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12386 break;
12387 }
12388
12389 default:
12390 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12391 VERR_VMX_UNEXPECTED_EXCEPTION);
12392 }
12393
12394 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12395 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12396 NOREF(pVM);
12397 return rcStrict;
12398}
12399
12400
12401/**
12402 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12403 * VM-exit.
12404 */
12405HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12406{
12407 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12408 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12409
12410 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12411 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12412 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12413 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12414 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12415 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12416 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12417 AssertRCReturn(rc2, rc2);
12418
12419 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12420 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12421 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12422 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12423 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12424 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12425 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12426 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12427 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12428
12429 /* I/O operation lookup arrays. */
12430 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12431 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12432
12433 VBOXSTRICTRC rcStrict;
12434 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12435 uint32_t const cbInstr = pVmxTransient->cbInstr;
12436 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12437 PVM pVM = pVCpu->CTX_SUFF(pVM);
12438 if (fIOString)
12439 {
12440#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12441 See @bugref{5752#c158}. Should work now. */
12442 /*
12443 * INS/OUTS - I/O String instruction.
12444 *
12445 * Use instruction-information if available, otherwise fall back on
12446 * interpreting the instruction.
12447 */
12448 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12449 fIOWrite ? 'w' : 'r'));
12450 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12451 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12452 {
12453 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12454 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12455 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12456 AssertRCReturn(rc2, rc2);
12457 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12458 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12459 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12460 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12461 if (fIOWrite)
12462 {
12463 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12464 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12465 }
12466 else
12467 {
12468 /*
12469 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12470 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12471 * See Intel Instruction spec. for "INS".
12472 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12473 */
12474 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12475 }
12476 }
12477 else
12478 {
12479 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12480 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12481 AssertRCReturn(rc2, rc2);
12482 rcStrict = IEMExecOne(pVCpu);
12483 }
12484 /** @todo IEM needs to be setting these flags somehow. */
12485 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12486 fUpdateRipAlready = true;
12487#else
12488 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12489 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12490 if (RT_SUCCESS(rcStrict))
12491 {
12492 if (fIOWrite)
12493 {
12494 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12495 (DISCPUMODE)pDis->uAddrMode, cbValue);
12496 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12497 }
12498 else
12499 {
12500 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12501 (DISCPUMODE)pDis->uAddrMode, cbValue);
12502 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12503 }
12504 }
12505 else
12506 {
12507 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12508 pMixedCtx->rip));
12509 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12510 }
12511#endif
12512 }
12513 else
12514 {
12515 /*
12516 * IN/OUT - I/O instruction.
12517 */
12518 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12519 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12520 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12521 if (fIOWrite)
12522 {
12523 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12524 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12525 }
12526 else
12527 {
12528 uint32_t u32Result = 0;
12529 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12530 if (IOM_SUCCESS(rcStrict))
12531 {
12532 /* Save result of I/O IN instr. in AL/AX/EAX. */
12533 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12534 }
12535 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12536 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12537 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12538 }
12539 }
12540
12541 if (IOM_SUCCESS(rcStrict))
12542 {
12543 if (!fUpdateRipAlready)
12544 {
12545 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12546 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12547 }
12548
12549 /*
12550 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12551 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12552 */
12553 if (fIOString)
12554 {
12555 /** @todo Single-step for INS/OUTS with REP prefix? */
12556 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12557 }
12558 else if ( !fDbgStepping
12559 && fGstStepping)
12560 {
12561 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12562 }
12563
12564 /*
12565 * If any I/O breakpoints are armed, we need to check if one triggered
12566 * and take appropriate action.
12567 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12568 */
12569 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12570 AssertRCReturn(rc2, rc2);
12571
12572 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12573 * execution engines about whether hyper BPs and such are pending. */
12574 uint32_t const uDr7 = pMixedCtx->dr[7];
12575 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12576 && X86_DR7_ANY_RW_IO(uDr7)
12577 && (pMixedCtx->cr4 & X86_CR4_DE))
12578 || DBGFBpIsHwIoArmed(pVM)))
12579 {
12580 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12581
12582 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12583 VMMRZCallRing3Disable(pVCpu);
12584 HM_DISABLE_PREEMPT();
12585
12586 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12587
12588 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12589 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12590 {
12591 /* Raise #DB. */
12592 if (fIsGuestDbgActive)
12593 ASMSetDR6(pMixedCtx->dr[6]);
12594 if (pMixedCtx->dr[7] != uDr7)
12595 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12596
12597 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12598 }
12599 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12600 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12601 else if ( rcStrict2 != VINF_SUCCESS
12602 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12603 rcStrict = rcStrict2;
12604 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12605
12606 HM_RESTORE_PREEMPT();
12607 VMMRZCallRing3Enable(pVCpu);
12608 }
12609 }
12610
12611#ifdef VBOX_STRICT
12612 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12613 Assert(!fIOWrite);
12614 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12615 Assert(fIOWrite);
12616 else
12617 {
12618#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12619 * statuses, that the VMM device and some others may return. See
12620 * IOM_SUCCESS() for guidance. */
12621 AssertMsg( RT_FAILURE(rcStrict)
12622 || rcStrict == VINF_SUCCESS
12623 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12624 || rcStrict == VINF_EM_DBG_BREAKPOINT
12625 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12626 || rcStrict == VINF_EM_RAW_TO_R3
12627 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12628#endif
12629 }
12630#endif
12631
12632 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12633 return rcStrict;
12634}
12635
12636
12637/**
12638 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12639 * VM-exit.
12640 */
12641HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12642{
12643 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12644
12645 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12646 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12647 AssertRCReturn(rc, rc);
12648 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12649 {
12650 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12651 AssertRCReturn(rc, rc);
12652 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12653 {
12654 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12655
12656 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12657 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12658
12659 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12660 Assert(!pVCpu->hm.s.Event.fPending);
12661 pVCpu->hm.s.Event.fPending = true;
12662 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12663 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12664 AssertRCReturn(rc, rc);
12665 if (fErrorCodeValid)
12666 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12667 else
12668 pVCpu->hm.s.Event.u32ErrCode = 0;
12669 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12670 && uVector == X86_XCPT_PF)
12671 {
12672 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12673 }
12674
12675 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12676 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12677 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12678 }
12679 }
12680
12681 /* Fall back to the interpreter to emulate the task-switch. */
12682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12683 return VERR_EM_INTERPRETER;
12684}
12685
12686
12687/**
12688 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12689 */
12690HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12691{
12692 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12693 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12694 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12695 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12696 AssertRCReturn(rc, rc);
12697 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12698 return VINF_EM_DBG_STEPPED;
12699}
12700
12701
12702/**
12703 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12704 */
12705HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12706{
12707 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12708
12709 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12710
12711 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12712 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12713 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12714 {
12715 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12716 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12717 {
12718 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12719 return VERR_EM_INTERPRETER;
12720 }
12721 }
12722 else
12723 {
12724 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12725 rcStrict1 = VINF_SUCCESS;
12726 return rcStrict1;
12727 }
12728
12729#if 0
12730 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12731 * just sync the whole thing. */
12732 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12733#else
12734 /* Aggressive state sync. for now. */
12735 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12736 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12737 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12738#endif
12739 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12740 AssertRCReturn(rc, rc);
12741
12742 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12743 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12744 VBOXSTRICTRC rcStrict2;
12745 switch (uAccessType)
12746 {
12747 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12748 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12749 {
12750 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12751 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12752 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12753
12754 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12755 GCPhys &= PAGE_BASE_GC_MASK;
12756 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12757 PVM pVM = pVCpu->CTX_SUFF(pVM);
12758 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12759 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12760
12761 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12762 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12763 CPUMCTX2CORE(pMixedCtx), GCPhys);
12764 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12765 if ( rcStrict2 == VINF_SUCCESS
12766 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12767 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12768 {
12769 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12770 | HM_CHANGED_GUEST_RSP
12771 | HM_CHANGED_GUEST_RFLAGS
12772 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12773 rcStrict2 = VINF_SUCCESS;
12774 }
12775 break;
12776 }
12777
12778 default:
12779 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12780 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12781 break;
12782 }
12783
12784 if (rcStrict2 != VINF_SUCCESS)
12785 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12786 return rcStrict2;
12787}
12788
12789
12790/**
12791 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12792 * VM-exit.
12793 */
12794HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12795{
12796 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12797
12798 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12799 if (pVmxTransient->fWasGuestDebugStateActive)
12800 {
12801 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12802 HMVMX_RETURN_UNEXPECTED_EXIT();
12803 }
12804
12805 if ( !pVCpu->hm.s.fSingleInstruction
12806 && !pVmxTransient->fWasHyperDebugStateActive)
12807 {
12808 Assert(!DBGFIsStepping(pVCpu));
12809 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12810
12811 /* Don't intercept MOV DRx any more. */
12812 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12813 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12814 AssertRCReturn(rc, rc);
12815
12816 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12817 VMMRZCallRing3Disable(pVCpu);
12818 HM_DISABLE_PREEMPT();
12819
12820 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12821 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12822 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12823
12824 HM_RESTORE_PREEMPT();
12825 VMMRZCallRing3Enable(pVCpu);
12826
12827#ifdef VBOX_WITH_STATISTICS
12828 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12829 AssertRCReturn(rc, rc);
12830 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12831 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12832 else
12833 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12834#endif
12835 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12836 return VINF_SUCCESS;
12837 }
12838
12839 /*
12840 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12841 * Update the segment registers and DR7 from the CPU.
12842 */
12843 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12844 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12845 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12846 AssertRCReturn(rc, rc);
12847 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12848
12849 PVM pVM = pVCpu->CTX_SUFF(pVM);
12850 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12851 {
12852 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12853 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12854 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12855 if (RT_SUCCESS(rc))
12856 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12857 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12858 }
12859 else
12860 {
12861 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12862 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12863 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12864 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12865 }
12866
12867 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12868 if (RT_SUCCESS(rc))
12869 {
12870 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12871 AssertRCReturn(rc2, rc2);
12872 return VINF_SUCCESS;
12873 }
12874 return rc;
12875}
12876
12877
12878/**
12879 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12880 * Conditional VM-exit.
12881 */
12882HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12883{
12884 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12885 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12886
12887 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12888 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12889 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12890 {
12891 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12892 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12893 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12894 {
12895 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12896 return VERR_EM_INTERPRETER;
12897 }
12898 }
12899 else
12900 {
12901 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12902 rcStrict1 = VINF_SUCCESS;
12903 return rcStrict1;
12904 }
12905
12906 RTGCPHYS GCPhys = 0;
12907 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12908
12909#if 0
12910 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12911#else
12912 /* Aggressive state sync. for now. */
12913 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12914 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12915 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12916#endif
12917 AssertRCReturn(rc, rc);
12918
12919 /*
12920 * If we succeed, resume guest execution.
12921 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12922 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12923 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12924 * weird case. See @bugref{6043}.
12925 */
12926 PVM pVM = pVCpu->CTX_SUFF(pVM);
12927 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12928 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12929 if ( rcStrict2 == VINF_SUCCESS
12930 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12931 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12932 {
12933 /* Successfully handled MMIO operation. */
12934 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12935 | HM_CHANGED_GUEST_RSP
12936 | HM_CHANGED_GUEST_RFLAGS
12937 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12938 return VINF_SUCCESS;
12939 }
12940 return rcStrict2;
12941}
12942
12943
12944/**
12945 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12946 * VM-exit.
12947 */
12948HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12949{
12950 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12951 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12952
12953 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12954 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12955 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12956 {
12957 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12958 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12959 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12960 }
12961 else
12962 {
12963 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12964 rcStrict1 = VINF_SUCCESS;
12965 return rcStrict1;
12966 }
12967
12968 RTGCPHYS GCPhys = 0;
12969 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12970 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12971#if 0
12972 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12973#else
12974 /* Aggressive state sync. for now. */
12975 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12976 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12977 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12978#endif
12979 AssertRCReturn(rc, rc);
12980
12981 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12982 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12983
12984 RTGCUINT uErrorCode = 0;
12985 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12986 uErrorCode |= X86_TRAP_PF_ID;
12987 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12988 uErrorCode |= X86_TRAP_PF_RW;
12989 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12990 uErrorCode |= X86_TRAP_PF_P;
12991
12992 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12993
12994 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12995 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12996
12997 /* Handle the pagefault trap for the nested shadow table. */
12998 PVM pVM = pVCpu->CTX_SUFF(pVM);
12999 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13000 TRPMResetTrap(pVCpu);
13001
13002 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13003 if ( rcStrict2 == VINF_SUCCESS
13004 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13005 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13006 {
13007 /* Successfully synced our nested page tables. */
13008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13009 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13010 | HM_CHANGED_GUEST_RSP
13011 | HM_CHANGED_GUEST_RFLAGS);
13012 return VINF_SUCCESS;
13013 }
13014
13015 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13016 return rcStrict2;
13017}
13018
13019/** @} */
13020
13021/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13022/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13023/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13024
13025/** @name VM-exit exception handlers.
13026 * @{
13027 */
13028
13029/**
13030 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13031 */
13032static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13033{
13034 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13035 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13036
13037 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13038 AssertRCReturn(rc, rc);
13039
13040 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13041 {
13042 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13043 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13044
13045 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13046 * provides VM-exit instruction length. If this causes problem later,
13047 * disassemble the instruction like it's done on AMD-V. */
13048 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13049 AssertRCReturn(rc2, rc2);
13050 return rc;
13051 }
13052
13053 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13054 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13055 return rc;
13056}
13057
13058
13059/**
13060 * VM-exit exception handler for \#BP (Breakpoint exception).
13061 */
13062static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13063{
13064 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13065 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13066
13067 /** @todo Try optimize this by not saving the entire guest state unless
13068 * really needed. */
13069 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13070 AssertRCReturn(rc, rc);
13071
13072 PVM pVM = pVCpu->CTX_SUFF(pVM);
13073 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13074 if (rc == VINF_EM_RAW_GUEST_TRAP)
13075 {
13076 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13077 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13078 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13079 AssertRCReturn(rc, rc);
13080
13081 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13082 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13083 }
13084
13085 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13086 return rc;
13087}
13088
13089
13090/**
13091 * VM-exit exception handler for \#AC (alignment check exception).
13092 */
13093static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13094{
13095 RT_NOREF_PV(pMixedCtx);
13096 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13097
13098 /*
13099 * Re-inject it. We'll detect any nesting before getting here.
13100 */
13101 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13102 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13103 AssertRCReturn(rc, rc);
13104 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13105
13106 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13107 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13108 return VINF_SUCCESS;
13109}
13110
13111
13112/**
13113 * VM-exit exception handler for \#DB (Debug exception).
13114 */
13115static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13116{
13117 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13118 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13119 Log6(("XcptDB\n"));
13120
13121 /*
13122 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13123 * for processing.
13124 */
13125 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13126 AssertRCReturn(rc, rc);
13127
13128 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13129 uint64_t uDR6 = X86_DR6_INIT_VAL;
13130 uDR6 |= ( pVmxTransient->uExitQualification
13131 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13132
13133 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13134 if (rc == VINF_EM_RAW_GUEST_TRAP)
13135 {
13136 /*
13137 * The exception was for the guest. Update DR6, DR7.GD and
13138 * IA32_DEBUGCTL.LBR before forwarding it.
13139 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13140 */
13141 VMMRZCallRing3Disable(pVCpu);
13142 HM_DISABLE_PREEMPT();
13143
13144 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13145 pMixedCtx->dr[6] |= uDR6;
13146 if (CPUMIsGuestDebugStateActive(pVCpu))
13147 ASMSetDR6(pMixedCtx->dr[6]);
13148
13149 HM_RESTORE_PREEMPT();
13150 VMMRZCallRing3Enable(pVCpu);
13151
13152 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13153 AssertRCReturn(rc, rc);
13154
13155 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13156 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13157
13158 /* Paranoia. */
13159 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13160 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13161
13162 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13163 AssertRCReturn(rc, rc);
13164
13165 /*
13166 * Raise #DB in the guest.
13167 *
13168 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13169 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13170 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13171 *
13172 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13173 */
13174 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13175 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13176 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13177 AssertRCReturn(rc, rc);
13178 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13179 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13180 return VINF_SUCCESS;
13181 }
13182
13183 /*
13184 * Not a guest trap, must be a hypervisor related debug event then.
13185 * Update DR6 in case someone is interested in it.
13186 */
13187 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13188 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13189 CPUMSetHyperDR6(pVCpu, uDR6);
13190
13191 return rc;
13192}
13193
13194
13195/**
13196 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13197 * point exception).
13198 */
13199static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13200{
13201 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13202
13203 /* We require CR0 and EFER. EFER is always up-to-date. */
13204 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13205 AssertRCReturn(rc, rc);
13206
13207 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13208 VMMRZCallRing3Disable(pVCpu);
13209 HM_DISABLE_PREEMPT();
13210
13211 /* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13212 if (pVmxTransient->fWasGuestFPUStateActive)
13213 {
13214 rc = VINF_EM_RAW_GUEST_TRAP;
13215 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13216 }
13217 else
13218 {
13219#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13220 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13221#endif
13222 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13223 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13224 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13225 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13226 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13227 }
13228
13229 HM_RESTORE_PREEMPT();
13230 VMMRZCallRing3Enable(pVCpu);
13231
13232 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13233 {
13234 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13235 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13236 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13237 pVCpu->hm.s.fPreloadGuestFpu = true;
13238 }
13239 else
13240 {
13241 /* Forward #NM to the guest. */
13242 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13243 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13244 AssertRCReturn(rc, rc);
13245 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13246 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13247 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13248 }
13249
13250 return VINF_SUCCESS;
13251}
13252
13253
13254/**
13255 * VM-exit exception handler for \#GP (General-protection exception).
13256 *
13257 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13258 */
13259static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13260{
13261 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13262 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13263
13264 int rc;
13265 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13266 { /* likely */ }
13267 else
13268 {
13269#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13270 Assert(pVCpu->hm.s.fUsingDebugLoop);
13271#endif
13272 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13273 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13274 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13275 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13276 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13277 AssertRCReturn(rc, rc);
13278 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13279 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13280 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13281 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13282 return rc;
13283 }
13284
13285 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13286 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13287
13288 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13289 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13290 AssertRCReturn(rc, rc);
13291
13292 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13293 uint32_t cbOp = 0;
13294 PVM pVM = pVCpu->CTX_SUFF(pVM);
13295 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13296 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13297 if (RT_SUCCESS(rc))
13298 {
13299 rc = VINF_SUCCESS;
13300 Assert(cbOp == pDis->cbInstr);
13301 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13302 switch (pDis->pCurInstr->uOpcode)
13303 {
13304 case OP_CLI:
13305 {
13306 pMixedCtx->eflags.Bits.u1IF = 0;
13307 pMixedCtx->eflags.Bits.u1RF = 0;
13308 pMixedCtx->rip += pDis->cbInstr;
13309 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13310 if ( !fDbgStepping
13311 && pMixedCtx->eflags.Bits.u1TF)
13312 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13314 break;
13315 }
13316
13317 case OP_STI:
13318 {
13319 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13320 pMixedCtx->eflags.Bits.u1IF = 1;
13321 pMixedCtx->eflags.Bits.u1RF = 0;
13322 pMixedCtx->rip += pDis->cbInstr;
13323 if (!fOldIF)
13324 {
13325 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13326 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13327 }
13328 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13329 if ( !fDbgStepping
13330 && pMixedCtx->eflags.Bits.u1TF)
13331 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13332 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13333 break;
13334 }
13335
13336 case OP_HLT:
13337 {
13338 rc = VINF_EM_HALT;
13339 pMixedCtx->rip += pDis->cbInstr;
13340 pMixedCtx->eflags.Bits.u1RF = 0;
13341 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13342 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13343 break;
13344 }
13345
13346 case OP_POPF:
13347 {
13348 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13349 uint32_t cbParm;
13350 uint32_t uMask;
13351 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13352 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13353 {
13354 cbParm = 4;
13355 uMask = 0xffffffff;
13356 }
13357 else
13358 {
13359 cbParm = 2;
13360 uMask = 0xffff;
13361 }
13362
13363 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13364 RTGCPTR GCPtrStack = 0;
13365 X86EFLAGS Eflags;
13366 Eflags.u32 = 0;
13367 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13368 &GCPtrStack);
13369 if (RT_SUCCESS(rc))
13370 {
13371 Assert(sizeof(Eflags.u32) >= cbParm);
13372 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13373 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13374 }
13375 if (RT_FAILURE(rc))
13376 {
13377 rc = VERR_EM_INTERPRETER;
13378 break;
13379 }
13380 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13381 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13382 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13383 pMixedCtx->esp += cbParm;
13384 pMixedCtx->esp &= uMask;
13385 pMixedCtx->rip += pDis->cbInstr;
13386 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13387 | HM_CHANGED_GUEST_RSP
13388 | HM_CHANGED_GUEST_RFLAGS);
13389 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13390 POPF restores EFLAGS.TF. */
13391 if ( !fDbgStepping
13392 && fGstStepping)
13393 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13394 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13395 break;
13396 }
13397
13398 case OP_PUSHF:
13399 {
13400 uint32_t cbParm;
13401 uint32_t uMask;
13402 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13403 {
13404 cbParm = 4;
13405 uMask = 0xffffffff;
13406 }
13407 else
13408 {
13409 cbParm = 2;
13410 uMask = 0xffff;
13411 }
13412
13413 /* Get the stack pointer & push the contents of eflags onto the stack. */
13414 RTGCPTR GCPtrStack = 0;
13415 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13416 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13417 if (RT_FAILURE(rc))
13418 {
13419 rc = VERR_EM_INTERPRETER;
13420 break;
13421 }
13422 X86EFLAGS Eflags = pMixedCtx->eflags;
13423 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13424 Eflags.Bits.u1RF = 0;
13425 Eflags.Bits.u1VM = 0;
13426
13427 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13428 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13429 {
13430 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13431 rc = VERR_EM_INTERPRETER;
13432 break;
13433 }
13434 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13435 pMixedCtx->esp -= cbParm;
13436 pMixedCtx->esp &= uMask;
13437 pMixedCtx->rip += pDis->cbInstr;
13438 pMixedCtx->eflags.Bits.u1RF = 0;
13439 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13440 | HM_CHANGED_GUEST_RSP
13441 | HM_CHANGED_GUEST_RFLAGS);
13442 if ( !fDbgStepping
13443 && pMixedCtx->eflags.Bits.u1TF)
13444 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13445 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13446 break;
13447 }
13448
13449 case OP_IRET:
13450 {
13451 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13452 * instruction reference. */
13453 RTGCPTR GCPtrStack = 0;
13454 uint32_t uMask = 0xffff;
13455 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13456 uint16_t aIretFrame[3];
13457 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13458 {
13459 rc = VERR_EM_INTERPRETER;
13460 break;
13461 }
13462 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13463 &GCPtrStack);
13464 if (RT_SUCCESS(rc))
13465 {
13466 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13467 PGMACCESSORIGIN_HM));
13468 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13469 }
13470 if (RT_FAILURE(rc))
13471 {
13472 rc = VERR_EM_INTERPRETER;
13473 break;
13474 }
13475 pMixedCtx->eip = 0;
13476 pMixedCtx->ip = aIretFrame[0];
13477 pMixedCtx->cs.Sel = aIretFrame[1];
13478 pMixedCtx->cs.ValidSel = aIretFrame[1];
13479 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13480 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13481 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13482 pMixedCtx->sp += sizeof(aIretFrame);
13483 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13484 | HM_CHANGED_GUEST_SEGMENT_REGS
13485 | HM_CHANGED_GUEST_RSP
13486 | HM_CHANGED_GUEST_RFLAGS);
13487 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13488 if ( !fDbgStepping
13489 && fGstStepping)
13490 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13491 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13492 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13493 break;
13494 }
13495
13496 case OP_INT:
13497 {
13498 uint16_t uVector = pDis->Param1.uValue & 0xff;
13499 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13500 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13501 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13502 break;
13503 }
13504
13505 case OP_INTO:
13506 {
13507 if (pMixedCtx->eflags.Bits.u1OF)
13508 {
13509 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13510 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13511 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13512 }
13513 else
13514 {
13515 pMixedCtx->eflags.Bits.u1RF = 0;
13516 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13517 }
13518 break;
13519 }
13520
13521 default:
13522 {
13523 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13524 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13525 EMCODETYPE_SUPERVISOR);
13526 rc = VBOXSTRICTRC_VAL(rc2);
13527 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13528 /** @todo We have to set pending-debug exceptions here when the guest is
13529 * single-stepping depending on the instruction that was interpreted. */
13530 Log4(("#GP rc=%Rrc\n", rc));
13531 break;
13532 }
13533 }
13534 }
13535 else
13536 rc = VERR_EM_INTERPRETER;
13537
13538 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13539 ("#GP Unexpected rc=%Rrc\n", rc));
13540 return rc;
13541}
13542
13543
13544/**
13545 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13546 * the exception reported in the VMX transient structure back into the VM.
13547 *
13548 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13549 * up-to-date.
13550 */
13551static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13552{
13553 RT_NOREF_PV(pMixedCtx);
13554 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13555#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13556 Assert(pVCpu->hm.s.fUsingDebugLoop);
13557#endif
13558
13559 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13560 hmR0VmxCheckExitDueToEventDelivery(). */
13561 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13562 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13563 AssertRCReturn(rc, rc);
13564 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13565
13566#ifdef DEBUG_ramshankar
13567 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13568 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13569 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13570#endif
13571
13572 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13573 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13574 return VINF_SUCCESS;
13575}
13576
13577
13578/**
13579 * VM-exit exception handler for \#PF (Page-fault exception).
13580 */
13581static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13582{
13583 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13584 PVM pVM = pVCpu->CTX_SUFF(pVM);
13585 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13586 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13587 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13588 AssertRCReturn(rc, rc);
13589
13590 if (!pVM->hm.s.fNestedPaging)
13591 { /* likely */ }
13592 else
13593 {
13594#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13595 Assert(pVCpu->hm.s.fUsingDebugLoop);
13596#endif
13597 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13598 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13599 {
13600 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13601 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13602 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13603 }
13604 else
13605 {
13606 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13607 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13608 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13609 }
13610 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13611 return rc;
13612 }
13613
13614 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13615 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13616 if (pVmxTransient->fVectoringPF)
13617 {
13618 Assert(pVCpu->hm.s.Event.fPending);
13619 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13620 }
13621
13622 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13623 AssertRCReturn(rc, rc);
13624
13625 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13626 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13627
13628 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13629 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13630 (RTGCPTR)pVmxTransient->uExitQualification);
13631
13632 Log4(("#PF: rc=%Rrc\n", rc));
13633 if (rc == VINF_SUCCESS)
13634 {
13635#if 0
13636 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13637 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13638 * memory? We don't update the whole state here... */
13639 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13640 | HM_CHANGED_GUEST_RSP
13641 | HM_CHANGED_GUEST_RFLAGS
13642 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13643#else
13644 /*
13645 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13646 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13647 */
13648 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13649 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13650#endif
13651 TRPMResetTrap(pVCpu);
13652 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13653 return rc;
13654 }
13655
13656 if (rc == VINF_EM_RAW_GUEST_TRAP)
13657 {
13658 if (!pVmxTransient->fVectoringDoublePF)
13659 {
13660 /* It's a guest page fault and needs to be reflected to the guest. */
13661 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13662 TRPMResetTrap(pVCpu);
13663 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13664 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13665 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13666 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13667 }
13668 else
13669 {
13670 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13671 TRPMResetTrap(pVCpu);
13672 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13673 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13674 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13675 }
13676
13677 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13678 return VINF_SUCCESS;
13679 }
13680
13681 TRPMResetTrap(pVCpu);
13682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13683 return rc;
13684}
13685
13686/** @} */
13687
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette