VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 73606

Last change on this file since 73606 was 73606, checked in by vboxsync, 7 years ago

VMM: Nested VMX: bugref:9180 Various bits:

  • IEM: Started VMXON, VMXOFF implementation, use IEM_OPCODE_GET_NEXT_RM.
  • IEM: Fixed INVPCID C impl, removed unused IEMExecDecodedInvpcid.
  • IEM: Updated iemCImpl_load_CrX to check for CR0/CR4 fixed bits in VMX.
  • IEM: Update offModRm to reset/re-initialize where needed.
  • CPUM: Added VMX root, non-root mode and other bits and updated a few places where they're used.
  • HM: Started adding fine-grained VMX instruction failure diagnostics.
  • HM: Made VM instruction error an enum.
  • HM: Added HMVMXAll.cpp for all context VMX code.
  • Ensure building with VBOX_WITH_NESTED_HWVIRT_[SVM|VMX] does the right thing based on host CPU.
  • CPUM: Added dumping of nested-VMX CPUMCTX state.
  • HMVMXR0: Added memory operand decoding.
  • HMVMXR0: VMX instr. privilege checks (CR0/CR4 read shadows are not consulted, so we need to do them)
  • HM: Added some more bit-field representaions.
  • Recompiler: Refuse to run when in nested-VMX guest code.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 580.3 KB
Line 
1/* $Id: HMVMXR0.cpp 73606 2018-08-10 07:38:56Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/em.h>
35#include <VBox/vmm/gim.h>
36#include <VBox/vmm/apic.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name HMVMX_READ_XXX
70 * Flags to skip redundant reads of some common VMCS fields that are not part of
71 * the guest-CPU or VCPU state but are needed while handling VM-exits.
72 */
73#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80/** @} */
81
82/**
83 * States of the VMCS.
84 *
85 * This does not reflect all possible VMCS states but currently only those
86 * needed for maintaining the VMCS consistently even when thread-context hooks
87 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
88 */
89#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
90#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
91#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
92
93/**
94 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
95 * guest using hardware-assisted VMX.
96 *
97 * This excludes state like GPRs (other than RSP) which are always are
98 * swapped and restored across the world-switch and also registers like EFER,
99 * MSR which cannot be modified by the guest without causing a VM-exit.
100 */
101#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
102 | CPUMCTX_EXTRN_RFLAGS \
103 | CPUMCTX_EXTRN_RSP \
104 | CPUMCTX_EXTRN_SREG_MASK \
105 | CPUMCTX_EXTRN_TABLE_MASK \
106 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
107 | CPUMCTX_EXTRN_SYSCALL_MSRS \
108 | CPUMCTX_EXTRN_SYSENTER_MSRS \
109 | CPUMCTX_EXTRN_TSC_AUX \
110 | CPUMCTX_EXTRN_OTHER_MSRS \
111 | CPUMCTX_EXTRN_CR0 \
112 | CPUMCTX_EXTRN_CR3 \
113 | CPUMCTX_EXTRN_CR4 \
114 | CPUMCTX_EXTRN_DR7 \
115 | CPUMCTX_EXTRN_HM_VMX_MASK)
116
117/**
118 * Exception bitmap mask for real-mode guests (real-on-v86).
119 *
120 * We need to intercept all exceptions manually except:
121 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
122 * due to bugs in Intel CPUs.
123 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
124 * support.
125 */
126#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
127 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
128 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
129 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
130 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
131 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
132 | RT_BIT(X86_XCPT_XF))
133
134/** Maximum VM-instruction error number. */
135#define HMVMX_INSTR_ERROR_MAX 28
136
137/** Profiling macro. */
138#ifdef HM_PROFILE_EXIT_DISPATCH
139# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
140# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
141#else
142# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
143# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
144#endif
145
146/** Assert that preemption is disabled or covered by thread-context hooks. */
147#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
149
150/** Assert that we haven't migrated CPUs when thread-context hooks are not
151 * used. */
152#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
153 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
154 ("Illegal migration! Entered on CPU %u Current %u\n", \
155 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
156
157/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
158 * context. */
159#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
160 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
161 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
162
163/** Helper macro for VM-exit handlers called unexpectedly. */
164#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
165 do { \
166 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
167 return VERR_VMX_UNEXPECTED_EXIT; \
168 } while (0)
169
170/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
171#ifdef VMX_USE_CACHED_VMCS_ACCESSES
172# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
173 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
174 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
175#else
176# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
177 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
178 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
179#endif
180
181/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
182# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
183 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
184 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
185
186
187/*********************************************************************************************************************************
188* Structures and Typedefs *
189*********************************************************************************************************************************/
190/**
191 * VMX transient state.
192 *
193 * A state structure for holding miscellaneous information across
194 * VMX non-root operation and restored after the transition.
195 */
196typedef struct VMXTRANSIENT
197{
198 /** The host's rflags/eflags. */
199 RTCCUINTREG fEFlags;
200#if HC_ARCH_BITS == 32
201 uint32_t u32Alignment0;
202#endif
203 /** The guest's TPR value used for TPR shadowing. */
204 uint8_t u8GuestTpr;
205 /** Alignment. */
206 uint8_t abAlignment0[7];
207
208 /** The basic VM-exit reason. */
209 uint16_t uExitReason;
210 /** Alignment. */
211 uint16_t u16Alignment0;
212 /** The VM-exit interruption error code. */
213 uint32_t uExitIntErrorCode;
214 /** The VM-exit exit code qualification. */
215 uint64_t uExitQual;
216
217 /** The VM-exit interruption-information field. */
218 uint32_t uExitIntInfo;
219 /** The VM-exit instruction-length field. */
220 uint32_t cbInstr;
221 /** The VM-exit instruction-information field. */
222 VMXEXITINSTRINFO ExitInstrInfo;
223 /** Whether the VM-entry failed or not. */
224 bool fVMEntryFailed;
225 /** Alignment. */
226 uint8_t abAlignment1[3];
227
228 /** The VM-entry interruption-information field. */
229 uint32_t uEntryIntInfo;
230 /** The VM-entry exception error code field. */
231 uint32_t uEntryXcptErrorCode;
232 /** The VM-entry instruction length field. */
233 uint32_t cbEntryInstr;
234
235 /** IDT-vectoring information field. */
236 uint32_t uIdtVectoringInfo;
237 /** IDT-vectoring error code. */
238 uint32_t uIdtVectoringErrorCode;
239
240 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
241 uint32_t fVmcsFieldsRead;
242
243 /** Whether the guest debug state was active at the time of VM-exit. */
244 bool fWasGuestDebugStateActive;
245 /** Whether the hyper debug state was active at the time of VM-exit. */
246 bool fWasHyperDebugStateActive;
247 /** Whether TSC-offsetting should be setup before VM-entry. */
248 bool fUpdateTscOffsettingAndPreemptTimer;
249 /** Whether the VM-exit was caused by a page-fault during delivery of a
250 * contributory exception or a page-fault. */
251 bool fVectoringDoublePF;
252 /** Whether the VM-exit was caused by a page-fault during delivery of an
253 * external interrupt or NMI. */
254 bool fVectoringPF;
255} VMXTRANSIENT;
256AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
257AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
258AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
259AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
260AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
261/** Pointer to VMX transient state. */
262typedef VMXTRANSIENT *PVMXTRANSIENT;
263
264
265/**
266 * MSR-bitmap read permissions.
267 */
268typedef enum VMXMSREXITREAD
269{
270 /** Reading this MSR causes a VM-exit. */
271 VMXMSREXIT_INTERCEPT_READ = 0xb,
272 /** Reading this MSR does not cause a VM-exit. */
273 VMXMSREXIT_PASSTHRU_READ
274} VMXMSREXITREAD;
275/** Pointer to MSR-bitmap read permissions. */
276typedef VMXMSREXITREAD* PVMXMSREXITREAD;
277
278/**
279 * MSR-bitmap write permissions.
280 */
281typedef enum VMXMSREXITWRITE
282{
283 /** Writing to this MSR causes a VM-exit. */
284 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
285 /** Writing to this MSR does not cause a VM-exit. */
286 VMXMSREXIT_PASSTHRU_WRITE
287} VMXMSREXITWRITE;
288/** Pointer to MSR-bitmap write permissions. */
289typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
290
291
292/**
293 * VMX VM-exit handler.
294 *
295 * @returns Strict VBox status code (i.e. informational status codes too).
296 * @param pVCpu The cross context virtual CPU structure.
297 * @param pVmxTransient Pointer to the VMX-transient structure.
298 */
299#ifndef HMVMX_USE_FUNCTION_TABLE
300typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
301#else
302typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
303/** Pointer to VM-exit handler. */
304typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
305#endif
306
307/**
308 * VMX VM-exit handler, non-strict status code.
309 *
310 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
311 *
312 * @returns VBox status code, no informational status code returned.
313 * @param pVCpu The cross context virtual CPU structure.
314 * @param pVmxTransient Pointer to the VMX-transient structure.
315 *
316 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
317 * use of that status code will be replaced with VINF_EM_SOMETHING
318 * later when switching over to IEM.
319 */
320#ifndef HMVMX_USE_FUNCTION_TABLE
321typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
322#else
323typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
324#endif
325
326
327/*********************************************************************************************************************************
328* Internal Functions *
329*********************************************************************************************************************************/
330static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
331static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
332static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
333static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
334static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
335 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
336#if HC_ARCH_BITS == 32
337static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
338#endif
339#ifndef HMVMX_USE_FUNCTION_TABLE
340DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
341# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
342# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
343#else
344# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
345# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
346#endif
347
348
349/** @name VM-exit handlers.
350 * @{
351 */
352static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
353static FNVMXEXITHANDLER hmR0VmxExitExtInt;
354static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
355static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
356static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
357static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
358static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
359static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
360static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
361static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
362static FNVMXEXITHANDLER hmR0VmxExitCpuid;
363static FNVMXEXITHANDLER hmR0VmxExitGetsec;
364static FNVMXEXITHANDLER hmR0VmxExitHlt;
365static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
366static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
367static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
368static FNVMXEXITHANDLER hmR0VmxExitVmcall;
369#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
370static FNVMXEXITHANDLER hmR0VmxExitVmclear;
371static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
372static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
373static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
374static FNVMXEXITHANDLER hmR0VmxExitVmread;
375static FNVMXEXITHANDLER hmR0VmxExitVmresume;
376static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
377static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
378static FNVMXEXITHANDLER hmR0VmxExitVmxon;
379#endif
380static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
381static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
382static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
383static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
384static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
385static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
386static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
387static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
391static FNVMXEXITHANDLER hmR0VmxExitMwait;
392static FNVMXEXITHANDLER hmR0VmxExitMtf;
393static FNVMXEXITHANDLER hmR0VmxExitMonitor;
394static FNVMXEXITHANDLER hmR0VmxExitPause;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
397static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
398static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
400static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
401static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
402static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
403static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
405static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
406static FNVMXEXITHANDLER hmR0VmxExitRdrand;
407static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
408/** @} */
409
410static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
411static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
412static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
413static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
417static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
418
419
420/*********************************************************************************************************************************
421* Global Variables *
422*********************************************************************************************************************************/
423#ifdef HMVMX_USE_FUNCTION_TABLE
424
425/**
426 * VMX_EXIT dispatch table.
427 */
428static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
429{
430 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
431 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
432 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
433 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
434 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
435 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
436 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
437 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
438 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
439 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
440 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
441 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
442 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
443 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
444 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
445 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
446 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
447 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
448 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
449#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
450 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
451 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
452 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
453 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
454 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
455 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
456 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
457 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
458 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
459#else
460 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
461 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
462 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
463 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
464 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
465 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
466 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
467 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
468 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
469#endif
470 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
471 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
472 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
473 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
474 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
475 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
476 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
477 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
478 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
479 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
480 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
481 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
482 /* 40 UNDEFINED */ hmR0VmxExitPause,
483 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
484 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
485 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
486 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
487 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
488 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
489 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
490 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
491 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
492 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
493 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
494 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
495 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
496 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
497 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
498 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
499 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
500 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
501 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
502 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
503 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
504 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
505 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
506 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
507};
508#endif /* HMVMX_USE_FUNCTION_TABLE */
509
510#ifdef VBOX_STRICT
511static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
512{
513 /* 0 */ "(Not Used)",
514 /* 1 */ "VMCALL executed in VMX root operation.",
515 /* 2 */ "VMCLEAR with invalid physical address.",
516 /* 3 */ "VMCLEAR with VMXON pointer.",
517 /* 4 */ "VMLAUNCH with non-clear VMCS.",
518 /* 5 */ "VMRESUME with non-launched VMCS.",
519 /* 6 */ "VMRESUME after VMXOFF",
520 /* 7 */ "VM-entry with invalid control fields.",
521 /* 8 */ "VM-entry with invalid host state fields.",
522 /* 9 */ "VMPTRLD with invalid physical address.",
523 /* 10 */ "VMPTRLD with VMXON pointer.",
524 /* 11 */ "VMPTRLD with incorrect revision identifier.",
525 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
526 /* 13 */ "VMWRITE to read-only VMCS component.",
527 /* 14 */ "(Not Used)",
528 /* 15 */ "VMXON executed in VMX root operation.",
529 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
530 /* 17 */ "VM-entry with non-launched executing VMCS.",
531 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
532 /* 19 */ "VMCALL with non-clear VMCS.",
533 /* 20 */ "VMCALL with invalid VM-exit control fields.",
534 /* 21 */ "(Not Used)",
535 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
536 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
537 /* 24 */ "VMCALL with invalid SMM-monitor features.",
538 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
539 /* 26 */ "VM-entry with events blocked by MOV SS.",
540 /* 27 */ "(Not Used)",
541 /* 28 */ "Invalid operand to INVEPT/INVVPID."
542};
543#endif /* VBOX_STRICT */
544
545
546
547/**
548 * Updates the VM's last error record.
549 *
550 * If there was a VMX instruction error, reads the error data from the VMCS and
551 * updates VCPU's last error record as well.
552 *
553 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
554 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
555 * VERR_VMX_INVALID_VMCS_FIELD.
556 * @param rc The error code.
557 */
558static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
559{
560 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
561 || rc == VERR_VMX_UNABLE_TO_START_VM)
562 {
563 AssertPtrReturnVoid(pVCpu);
564 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
565 }
566 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
567}
568
569
570/**
571 * Reads the VM-entry interruption-information field from the VMCS into the VMX
572 * transient structure.
573 *
574 * @returns VBox status code.
575 * @param pVmxTransient Pointer to the VMX transient structure.
576 *
577 * @remarks No-long-jump zone!!!
578 */
579DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
580{
581 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
582 AssertRCReturn(rc, rc);
583 return VINF_SUCCESS;
584}
585
586#ifdef VBOX_STRICT
587/**
588 * Reads the VM-entry exception error code field from the VMCS into
589 * the VMX transient structure.
590 *
591 * @returns VBox status code.
592 * @param pVmxTransient Pointer to the VMX transient structure.
593 *
594 * @remarks No-long-jump zone!!!
595 */
596DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
597{
598 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
599 AssertRCReturn(rc, rc);
600 return VINF_SUCCESS;
601}
602
603
604/**
605 * Reads the VM-entry exception error code field from the VMCS into
606 * the VMX transient structure.
607 *
608 * @returns VBox status code.
609 * @param pVmxTransient Pointer to the VMX transient structure.
610 *
611 * @remarks No-long-jump zone!!!
612 */
613DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
614{
615 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
616 AssertRCReturn(rc, rc);
617 return VINF_SUCCESS;
618}
619#endif /* VBOX_STRICT */
620
621
622/**
623 * Reads the VM-exit interruption-information field from the VMCS into the VMX
624 * transient structure.
625 *
626 * @returns VBox status code.
627 * @param pVmxTransient Pointer to the VMX transient structure.
628 */
629DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
630{
631 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
632 {
633 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
634 AssertRCReturn(rc,rc);
635 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
636 }
637 return VINF_SUCCESS;
638}
639
640
641/**
642 * Reads the VM-exit interruption error code from the VMCS into the VMX
643 * transient structure.
644 *
645 * @returns VBox status code.
646 * @param pVmxTransient Pointer to the VMX transient structure.
647 */
648DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
649{
650 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
651 {
652 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
653 AssertRCReturn(rc, rc);
654 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
655 }
656 return VINF_SUCCESS;
657}
658
659
660/**
661 * Reads the VM-exit instruction length field from the VMCS into the VMX
662 * transient structure.
663 *
664 * @returns VBox status code.
665 * @param pVmxTransient Pointer to the VMX transient structure.
666 */
667DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
668{
669 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
670 {
671 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
672 AssertRCReturn(rc, rc);
673 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
674 }
675 return VINF_SUCCESS;
676}
677
678
679/**
680 * Reads the VM-exit instruction-information field from the VMCS into
681 * the VMX transient structure.
682 *
683 * @returns VBox status code.
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
689 {
690 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the exit code qualification from the VMCS into the VMX transient
700 * structure.
701 *
702 * @returns VBox status code.
703 * @param pVCpu The cross context virtual CPU structure of the
704 * calling EMT. (Required for the VMCS cache case.)
705 * @param pVmxTransient Pointer to the VMX transient structure.
706 */
707DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
710 {
711 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the IDT-vectoring information field from the VMCS into the VMX
721 * transient structure.
722 *
723 * @returns VBox status code.
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 *
726 * @remarks No-long-jump zone!!!
727 */
728DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
729{
730 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
731 {
732 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
733 AssertRCReturn(rc, rc);
734 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
735 }
736 return VINF_SUCCESS;
737}
738
739
740/**
741 * Reads the IDT-vectoring error code from the VMCS into the VMX
742 * transient structure.
743 *
744 * @returns VBox status code.
745 * @param pVmxTransient Pointer to the VMX transient structure.
746 */
747DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
748{
749 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
750 {
751 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
752 AssertRCReturn(rc, rc);
753 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
754 }
755 return VINF_SUCCESS;
756}
757
758
759/**
760 * Enters VMX root mode operation on the current CPU.
761 *
762 * @returns VBox status code.
763 * @param pVM The cross context VM structure. Can be
764 * NULL, after a resume.
765 * @param HCPhysCpuPage Physical address of the VMXON region.
766 * @param pvCpuPage Pointer to the VMXON region.
767 */
768static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
769{
770 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
771 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
772 Assert(pvCpuPage);
773 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
774
775 if (pVM)
776 {
777 /* Write the VMCS revision dword to the VMXON region. */
778 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
779 }
780
781 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
782 RTCCUINTREG fEFlags = ASMIntDisableFlags();
783
784 /* Enable the VMX bit in CR4 if necessary. */
785 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
786
787 /* Enter VMX root mode. */
788 int rc = VMXEnable(HCPhysCpuPage);
789 if (RT_FAILURE(rc))
790 {
791 if (!(uOldCr4 & X86_CR4_VMXE))
792 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
793
794 if (pVM)
795 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
796 }
797
798 /* Restore interrupts. */
799 ASMSetFlags(fEFlags);
800 return rc;
801}
802
803
804/**
805 * Exits VMX root mode operation on the current CPU.
806 *
807 * @returns VBox status code.
808 */
809static int hmR0VmxLeaveRootMode(void)
810{
811 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
812
813 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
814 RTCCUINTREG fEFlags = ASMIntDisableFlags();
815
816 /* If we're for some reason not in VMX root mode, then don't leave it. */
817 RTCCUINTREG uHostCR4 = ASMGetCR4();
818
819 int rc;
820 if (uHostCR4 & X86_CR4_VMXE)
821 {
822 /* Exit VMX root mode and clear the VMX bit in CR4. */
823 VMXDisable();
824 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
825 rc = VINF_SUCCESS;
826 }
827 else
828 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
829
830 /* Restore interrupts. */
831 ASMSetFlags(fEFlags);
832 return rc;
833}
834
835
836/**
837 * Allocates and maps one physically contiguous page. The allocated page is
838 * zero'd out. (Used by various VT-x structures).
839 *
840 * @returns IPRT status code.
841 * @param pMemObj Pointer to the ring-0 memory object.
842 * @param ppVirt Where to store the virtual address of the
843 * allocation.
844 * @param pHCPhys Where to store the physical address of the
845 * allocation.
846 */
847static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
848{
849 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
850 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
851 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
852
853 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
854 if (RT_FAILURE(rc))
855 return rc;
856 *ppVirt = RTR0MemObjAddress(*pMemObj);
857 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
858 ASMMemZero32(*ppVirt, PAGE_SIZE);
859 return VINF_SUCCESS;
860}
861
862
863/**
864 * Frees and unmaps an allocated physical page.
865 *
866 * @param pMemObj Pointer to the ring-0 memory object.
867 * @param ppVirt Where to re-initialize the virtual address of
868 * allocation as 0.
869 * @param pHCPhys Where to re-initialize the physical address of the
870 * allocation as 0.
871 */
872static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
873{
874 AssertPtr(pMemObj);
875 AssertPtr(ppVirt);
876 AssertPtr(pHCPhys);
877 if (*pMemObj != NIL_RTR0MEMOBJ)
878 {
879 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
880 AssertRC(rc);
881 *pMemObj = NIL_RTR0MEMOBJ;
882 *ppVirt = 0;
883 *pHCPhys = 0;
884 }
885}
886
887
888/**
889 * Worker function to free VT-x related structures.
890 *
891 * @returns IPRT status code.
892 * @param pVM The cross context VM structure.
893 */
894static void hmR0VmxStructsFree(PVM pVM)
895{
896 for (VMCPUID i = 0; i < pVM->cCpus; i++)
897 {
898 PVMCPU pVCpu = &pVM->aCpus[i];
899 AssertPtr(pVCpu);
900
901 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
902 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
903
904 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
905 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
906
907 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
908 }
909
910 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
911#ifdef VBOX_WITH_CRASHDUMP_MAGIC
912 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
913#endif
914}
915
916
917/**
918 * Worker function to allocate VT-x related VM structures.
919 *
920 * @returns IPRT status code.
921 * @param pVM The cross context VM structure.
922 */
923static int hmR0VmxStructsAlloc(PVM pVM)
924{
925 /*
926 * Initialize members up-front so we can cleanup properly on allocation failure.
927 */
928#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
929 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
930 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
931 pVM->hm.s.vmx.HCPhys##a_Name = 0;
932
933#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
934 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
935 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
936 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
937
938#ifdef VBOX_WITH_CRASHDUMP_MAGIC
939 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
940#endif
941 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
942
943 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
944 for (VMCPUID i = 0; i < pVM->cCpus; i++)
945 {
946 PVMCPU pVCpu = &pVM->aCpus[i];
947 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
948 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
949 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
950 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
951 }
952#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
953#undef VMXLOCAL_INIT_VM_MEMOBJ
954
955 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
956 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
957 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
958 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
959
960 /*
961 * Allocate all the VT-x structures.
962 */
963 int rc = VINF_SUCCESS;
964#ifdef VBOX_WITH_CRASHDUMP_MAGIC
965 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
966 if (RT_FAILURE(rc))
967 goto cleanup;
968 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
969 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
970#endif
971
972 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
973 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
974 {
975 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
976 &pVM->hm.s.vmx.HCPhysApicAccess);
977 if (RT_FAILURE(rc))
978 goto cleanup;
979 }
980
981 /*
982 * Initialize per-VCPU VT-x structures.
983 */
984 for (VMCPUID i = 0; i < pVM->cCpus; i++)
985 {
986 PVMCPU pVCpu = &pVM->aCpus[i];
987 AssertPtr(pVCpu);
988
989 /* Allocate the VM control structure (VMCS). */
990 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993
994 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
995 if ( PDMHasApic(pVM)
996 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
997 {
998 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
999 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1000 if (RT_FAILURE(rc))
1001 goto cleanup;
1002 }
1003
1004 /*
1005 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1006 * transparent accesses of specific MSRs.
1007 *
1008 * If the condition for enabling MSR bitmaps changes here, don't forget to
1009 * update HMAreMsrBitmapsAvailable().
1010 */
1011 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1012 {
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1014 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1018 }
1019
1020 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1021 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1022 if (RT_FAILURE(rc))
1023 goto cleanup;
1024
1025 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1027 if (RT_FAILURE(rc))
1028 goto cleanup;
1029 }
1030
1031 return VINF_SUCCESS;
1032
1033cleanup:
1034 hmR0VmxStructsFree(pVM);
1035 return rc;
1036}
1037
1038
1039/**
1040 * Does global VT-x initialization (called during module initialization).
1041 *
1042 * @returns VBox status code.
1043 */
1044VMMR0DECL(int) VMXR0GlobalInit(void)
1045{
1046#ifdef HMVMX_USE_FUNCTION_TABLE
1047 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1048# ifdef VBOX_STRICT
1049 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1050 Assert(g_apfnVMExitHandlers[i]);
1051# endif
1052#endif
1053 return VINF_SUCCESS;
1054}
1055
1056
1057/**
1058 * Does global VT-x termination (called during module termination).
1059 */
1060VMMR0DECL(void) VMXR0GlobalTerm()
1061{
1062 /* Nothing to do currently. */
1063}
1064
1065
1066/**
1067 * Sets up and activates VT-x on the current CPU.
1068 *
1069 * @returns VBox status code.
1070 * @param pHostCpu Pointer to the global CPU info struct.
1071 * @param pVM The cross context VM structure. Can be
1072 * NULL after a host resume operation.
1073 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1074 * fEnabledByHost is @c true).
1075 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1076 * @a fEnabledByHost is @c true).
1077 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1078 * enable VT-x on the host.
1079 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1080 */
1081VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1082 void *pvMsrs)
1083{
1084 Assert(pHostCpu);
1085 Assert(pvMsrs);
1086 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1087
1088 /* Enable VT-x if it's not already enabled by the host. */
1089 if (!fEnabledByHost)
1090 {
1091 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1092 if (RT_FAILURE(rc))
1093 return rc;
1094 }
1095
1096 /*
1097 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1098 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1099 * invalidated when flushing by VPID.
1100 */
1101 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1102 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1103 {
1104 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1105 pHostCpu->fFlushAsidBeforeUse = false;
1106 }
1107 else
1108 pHostCpu->fFlushAsidBeforeUse = true;
1109
1110 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1111 ++pHostCpu->cTlbFlushes;
1112
1113 return VINF_SUCCESS;
1114}
1115
1116
1117/**
1118 * Deactivates VT-x on the current CPU.
1119 *
1120 * @returns VBox status code.
1121 * @param pHostCpu Pointer to the global CPU info struct.
1122 * @param pvCpuPage Pointer to the VMXON region.
1123 * @param HCPhysCpuPage Physical address of the VMXON region.
1124 *
1125 * @remarks This function should never be called when SUPR0EnableVTx() or
1126 * similar was used to enable VT-x on the host.
1127 */
1128VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1129{
1130 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1131
1132 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1133 return hmR0VmxLeaveRootMode();
1134}
1135
1136
1137/**
1138 * Sets the permission bits for the specified MSR in the MSR bitmap.
1139 *
1140 * @param pVCpu The cross context virtual CPU structure.
1141 * @param uMsr The MSR value.
1142 * @param enmRead Whether reading this MSR causes a VM-exit.
1143 * @param enmWrite Whether writing this MSR causes a VM-exit.
1144 */
1145static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1146{
1147 int32_t iBit;
1148 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1149
1150 /*
1151 * Layout:
1152 * 0x000 - 0x3ff - Low MSR read bits
1153 * 0x400 - 0x7ff - High MSR read bits
1154 * 0x800 - 0xbff - Low MSR write bits
1155 * 0xc00 - 0xfff - High MSR write bits
1156 */
1157 if (uMsr <= 0x00001fff)
1158 iBit = uMsr;
1159 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1160 {
1161 iBit = uMsr - UINT32_C(0xc0000000);
1162 pbMsrBitmap += 0x400;
1163 }
1164 else
1165 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1166
1167 Assert(iBit <= 0x1fff);
1168 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1169 ASMBitSet(pbMsrBitmap, iBit);
1170 else
1171 ASMBitClear(pbMsrBitmap, iBit);
1172
1173 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1174 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1175 else
1176 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1177}
1178
1179
1180#ifdef VBOX_STRICT
1181/**
1182 * Gets the permission bits for the specified MSR in the MSR bitmap.
1183 *
1184 * @returns VBox status code.
1185 * @retval VINF_SUCCESS if the specified MSR is found.
1186 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1187 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1188 *
1189 * @param pVCpu The cross context virtual CPU structure.
1190 * @param uMsr The MSR.
1191 * @param penmRead Where to store the read permissions.
1192 * @param penmWrite Where to store the write permissions.
1193 */
1194static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1195{
1196 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1197 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1198 int32_t iBit;
1199 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1200
1201 /* See hmR0VmxSetMsrPermission() for the layout. */
1202 if (uMsr <= 0x00001fff)
1203 iBit = uMsr;
1204 else if ( uMsr >= 0xc0000000
1205 && uMsr <= 0xc0001fff)
1206 {
1207 iBit = (uMsr - 0xc0000000);
1208 pbMsrBitmap += 0x400;
1209 }
1210 else
1211 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1212
1213 Assert(iBit <= 0x1fff);
1214 if (ASMBitTest(pbMsrBitmap, iBit))
1215 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1216 else
1217 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1218
1219 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1220 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1221 else
1222 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1223 return VINF_SUCCESS;
1224}
1225#endif /* VBOX_STRICT */
1226
1227
1228/**
1229 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1230 * area.
1231 *
1232 * @returns VBox status code.
1233 * @param pVCpu The cross context virtual CPU structure.
1234 * @param cMsrs The number of MSRs.
1235 */
1236static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1237{
1238 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1239 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1240 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1241 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1242 {
1243 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1244 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1245 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1246 }
1247
1248 /* Update number of guest MSRs to load/store across the world-switch. */
1249 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1250 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1251
1252 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1253 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1254 AssertRCReturn(rc, rc);
1255
1256 /* Update the VCPU's copy of the MSR count. */
1257 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1258
1259 return VINF_SUCCESS;
1260}
1261
1262
1263/**
1264 * Adds a new (or updates the value of an existing) guest/host MSR
1265 * pair to be swapped during the world-switch as part of the
1266 * auto-load/store MSR area in the VMCS.
1267 *
1268 * @returns VBox status code.
1269 * @param pVCpu The cross context virtual CPU structure.
1270 * @param uMsr The MSR.
1271 * @param uGuestMsrValue Value of the guest MSR.
1272 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1273 * necessary.
1274 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1275 * its value was updated. Optional, can be NULL.
1276 */
1277static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1278 bool *pfAddedAndUpdated)
1279{
1280 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1281 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1282 uint32_t i;
1283 for (i = 0; i < cMsrs; i++)
1284 {
1285 if (pGuestMsr->u32Msr == uMsr)
1286 break;
1287 pGuestMsr++;
1288 }
1289
1290 bool fAdded = false;
1291 if (i == cMsrs)
1292 {
1293 ++cMsrs;
1294 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1295 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1296
1297 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1298 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1299 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1300
1301 fAdded = true;
1302 }
1303
1304 /* Update the MSR values in the auto-load/store MSR area. */
1305 pGuestMsr->u32Msr = uMsr;
1306 pGuestMsr->u64Value = uGuestMsrValue;
1307
1308 /* Create/update the MSR slot in the host MSR area. */
1309 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1310 pHostMsr += i;
1311 pHostMsr->u32Msr = uMsr;
1312
1313 /*
1314 * Update the host MSR only when requested by the caller AND when we're
1315 * adding it to the auto-load/store area. Otherwise, it would have been
1316 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1317 */
1318 bool fUpdatedMsrValue = false;
1319 if ( fAdded
1320 && fUpdateHostMsr)
1321 {
1322 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1323 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1324 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1325 fUpdatedMsrValue = true;
1326 }
1327
1328 if (pfAddedAndUpdated)
1329 *pfAddedAndUpdated = fUpdatedMsrValue;
1330 return VINF_SUCCESS;
1331}
1332
1333
1334/**
1335 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1336 * auto-load/store MSR area in the VMCS.
1337 *
1338 * @returns VBox status code.
1339 * @param pVCpu The cross context virtual CPU structure.
1340 * @param uMsr The MSR.
1341 */
1342static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1343{
1344 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1345 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1346 for (uint32_t i = 0; i < cMsrs; i++)
1347 {
1348 /* Find the MSR. */
1349 if (pGuestMsr->u32Msr == uMsr)
1350 {
1351 /* If it's the last MSR, simply reduce the count. */
1352 if (i == cMsrs - 1)
1353 {
1354 --cMsrs;
1355 break;
1356 }
1357
1358 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1359 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1360 pLastGuestMsr += cMsrs - 1;
1361 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1362 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1363
1364 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1365 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1366 pLastHostMsr += cMsrs - 1;
1367 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1368 pHostMsr->u64Value = pLastHostMsr->u64Value;
1369 --cMsrs;
1370 break;
1371 }
1372 pGuestMsr++;
1373 }
1374
1375 /* Update the VMCS if the count changed (meaning the MSR was found). */
1376 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1377 {
1378 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1379 AssertRCReturn(rc, rc);
1380
1381 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1382 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1383 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1384
1385 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1386 return VINF_SUCCESS;
1387 }
1388
1389 return VERR_NOT_FOUND;
1390}
1391
1392
1393/**
1394 * Checks if the specified guest MSR is part of the auto-load/store area in
1395 * the VMCS.
1396 *
1397 * @returns true if found, false otherwise.
1398 * @param pVCpu The cross context virtual CPU structure.
1399 * @param uMsr The MSR to find.
1400 */
1401static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1402{
1403 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1404 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1405
1406 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1407 {
1408 if (pGuestMsr->u32Msr == uMsr)
1409 return true;
1410 }
1411 return false;
1412}
1413
1414
1415/**
1416 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1417 *
1418 * @param pVCpu The cross context virtual CPU structure.
1419 *
1420 * @remarks No-long-jump zone!!!
1421 */
1422static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1423{
1424 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1425 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1426 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1427 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1428
1429 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1430 {
1431 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1432
1433 /*
1434 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1435 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1436 */
1437 if (pHostMsr->u32Msr == MSR_K6_EFER)
1438 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1439 else
1440 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1441 }
1442
1443 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1444}
1445
1446
1447/**
1448 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1449 * perform lazy restoration of the host MSRs while leaving VT-x.
1450 *
1451 * @param pVCpu The cross context virtual CPU structure.
1452 *
1453 * @remarks No-long-jump zone!!!
1454 */
1455static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1456{
1457 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1458
1459 /*
1460 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1461 */
1462 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1463 {
1464 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1465#if HC_ARCH_BITS == 64
1466 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1467 {
1468 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1469 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1470 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1471 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1472 }
1473#endif
1474 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1475 }
1476}
1477
1478
1479/**
1480 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1481 * lazily while leaving VT-x.
1482 *
1483 * @returns true if it does, false otherwise.
1484 * @param pVCpu The cross context virtual CPU structure.
1485 * @param uMsr The MSR to check.
1486 */
1487static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1488{
1489 NOREF(pVCpu);
1490#if HC_ARCH_BITS == 64
1491 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1492 {
1493 switch (uMsr)
1494 {
1495 case MSR_K8_LSTAR:
1496 case MSR_K6_STAR:
1497 case MSR_K8_SF_MASK:
1498 case MSR_K8_KERNEL_GS_BASE:
1499 return true;
1500 }
1501 }
1502#else
1503 RT_NOREF(pVCpu, uMsr);
1504#endif
1505 return false;
1506}
1507
1508
1509/**
1510 * Loads a set of guests MSRs to allow read/passthru to the guest.
1511 *
1512 * The name of this function is slightly confusing. This function does NOT
1513 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1514 * common prefix for functions dealing with "lazy restoration" of the shared
1515 * MSRs.
1516 *
1517 * @param pVCpu The cross context virtual CPU structure.
1518 *
1519 * @remarks No-long-jump zone!!!
1520 */
1521static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1522{
1523 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1524 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1525
1526 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1527#if HC_ARCH_BITS == 64
1528 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1529 {
1530 /*
1531 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1532 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1533 * we can skip a few MSR writes.
1534 *
1535 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1536 * guest MSR values in the guest-CPU context might be different to what's currently
1537 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1538 * CPU, see @bugref{8728}.
1539 */
1540 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1541 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1542 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1543 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1544 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1545 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1546 {
1547#ifdef VBOX_STRICT
1548 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1549 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1550 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1551 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1552#endif
1553 }
1554 else
1555 {
1556 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1557 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1558 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1559 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1560 }
1561 }
1562#endif
1563 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1564}
1565
1566
1567/**
1568 * Performs lazy restoration of the set of host MSRs if they were previously
1569 * loaded with guest MSR values.
1570 *
1571 * @param pVCpu The cross context virtual CPU structure.
1572 *
1573 * @remarks No-long-jump zone!!!
1574 * @remarks The guest MSRs should have been saved back into the guest-CPU
1575 * context by hmR0VmxImportGuestState()!!!
1576 */
1577static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1578{
1579 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1580 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1581
1582 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1583 {
1584 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1585#if HC_ARCH_BITS == 64
1586 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1587 {
1588 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1589 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1590 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1591 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1592 }
1593#endif
1594 }
1595 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1596}
1597
1598
1599/**
1600 * Verifies that our cached values of the VMCS fields are all consistent with
1601 * what's actually present in the VMCS.
1602 *
1603 * @returns VBox status code.
1604 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1605 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1606 * VMCS content. HMCPU error-field is
1607 * updated, see VMX_VCI_XXX.
1608 * @param pVCpu The cross context virtual CPU structure.
1609 */
1610static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1611{
1612 uint32_t u32Val;
1613 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1614 AssertRCReturn(rc, rc);
1615 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1616 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1617 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1618 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1619
1620 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1621 AssertRCReturn(rc, rc);
1622 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1623 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1624 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1625 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1626
1627 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1628 AssertRCReturn(rc, rc);
1629 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1630 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1631 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1632 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1633
1634 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1635 AssertRCReturn(rc, rc);
1636 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1637 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1638 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1639 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1640
1641 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1642 {
1643 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1644 AssertRCReturn(rc, rc);
1645 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1646 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1647 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1648 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1649 }
1650
1651 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1652 AssertRCReturn(rc, rc);
1653 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1654 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1655 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1656 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1657
1658 uint64_t u64Val;
1659 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1660 AssertRCReturn(rc, rc);
1661 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1662 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1663 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1664 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1665
1666 return VINF_SUCCESS;
1667}
1668
1669
1670#ifdef VBOX_STRICT
1671/**
1672 * Verifies that our cached host EFER value has not changed
1673 * since we cached it.
1674 *
1675 * @param pVCpu The cross context virtual CPU structure.
1676 */
1677static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1678{
1679 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1680
1681 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1682 {
1683 uint64_t u64Val;
1684 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1685 AssertRC(rc);
1686
1687 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1688 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1689 }
1690}
1691
1692
1693/**
1694 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1695 * VMCS are correct.
1696 *
1697 * @param pVCpu The cross context virtual CPU structure.
1698 */
1699static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1700{
1701 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1702
1703 /* Verify MSR counts in the VMCS are what we think it should be. */
1704 uint32_t cMsrs;
1705 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1706 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1707
1708 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1715 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1716 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1717 {
1718 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1719 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1720 pGuestMsr->u32Msr, cMsrs));
1721
1722 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1723 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1724 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1725
1726 /* Verify that the permissions are as expected in the MSR bitmap. */
1727 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1728 {
1729 VMXMSREXITREAD enmRead;
1730 VMXMSREXITWRITE enmWrite;
1731 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1732 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1733 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1734 {
1735 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1736 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1737 }
1738 else
1739 {
1740 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1741 pGuestMsr->u32Msr, cMsrs));
1742 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1743 pGuestMsr->u32Msr, cMsrs));
1744 }
1745 }
1746 }
1747}
1748#endif /* VBOX_STRICT */
1749
1750
1751/**
1752 * Flushes the TLB using EPT.
1753 *
1754 * @returns VBox status code.
1755 * @param pVCpu The cross context virtual CPU structure of the calling
1756 * EMT. Can be NULL depending on @a enmTlbFlush.
1757 * @param enmTlbFlush Type of flush.
1758 *
1759 * @remarks Caller is responsible for making sure this function is called only
1760 * when NestedPaging is supported and providing @a enmTlbFlush that is
1761 * supported by the CPU.
1762 * @remarks Can be called with interrupts disabled.
1763 */
1764static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1765{
1766 uint64_t au64Descriptor[2];
1767 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1768 au64Descriptor[0] = 0;
1769 else
1770 {
1771 Assert(pVCpu);
1772 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1773 }
1774 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1775
1776 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1777 AssertMsg(rc == VINF_SUCCESS,
1778 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1779
1780 if ( RT_SUCCESS(rc)
1781 && pVCpu)
1782 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1783}
1784
1785
1786/**
1787 * Flushes the TLB using VPID.
1788 *
1789 * @returns VBox status code.
1790 * @param pVCpu The cross context virtual CPU structure of the calling
1791 * EMT. Can be NULL depending on @a enmTlbFlush.
1792 * @param enmTlbFlush Type of flush.
1793 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1794 * on @a enmTlbFlush).
1795 *
1796 * @remarks Can be called with interrupts disabled.
1797 */
1798static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1799{
1800 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1801
1802 uint64_t au64Descriptor[2];
1803 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1804 {
1805 au64Descriptor[0] = 0;
1806 au64Descriptor[1] = 0;
1807 }
1808 else
1809 {
1810 AssertPtr(pVCpu);
1811 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1812 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1813 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1814 au64Descriptor[1] = GCPtr;
1815 }
1816
1817 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1818 AssertMsg(rc == VINF_SUCCESS,
1819 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1820
1821 if ( RT_SUCCESS(rc)
1822 && pVCpu)
1823 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1824 NOREF(rc);
1825}
1826
1827
1828/**
1829 * Invalidates a guest page by guest virtual address. Only relevant for
1830 * EPT/VPID, otherwise there is nothing really to invalidate.
1831 *
1832 * @returns VBox status code.
1833 * @param pVCpu The cross context virtual CPU structure.
1834 * @param GCVirt Guest virtual address of the page to invalidate.
1835 */
1836VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1837{
1838 AssertPtr(pVCpu);
1839 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1840
1841 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1842 if (!fFlushPending)
1843 {
1844 /*
1845 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1846 * the EPT case. See @bugref{6043} and @bugref{6177}.
1847 *
1848 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1849 * as this function maybe called in a loop with individual addresses.
1850 */
1851 PVM pVM = pVCpu->CTX_SUFF(pVM);
1852 if (pVM->hm.s.vmx.fVpid)
1853 {
1854 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1855
1856#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1857 /*
1858 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1859 * where executing INVVPID outside 64-bit mode does not flush translations of
1860 * 64-bit linear addresses, see @bugref{6208#c72}.
1861 */
1862 if (RT_HI_U32(GCVirt))
1863 fVpidFlush = false;
1864#endif
1865
1866 if (fVpidFlush)
1867 {
1868 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1869 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1870 }
1871 else
1872 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1873 }
1874 else if (pVM->hm.s.fNestedPaging)
1875 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1876 }
1877
1878 return VINF_SUCCESS;
1879}
1880
1881
1882/**
1883 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1884 * case where neither EPT nor VPID is supported by the CPU.
1885 *
1886 * @param pVCpu The cross context virtual CPU structure.
1887 * @param pCpu Pointer to the global HM struct.
1888 *
1889 * @remarks Called with interrupts disabled.
1890 */
1891static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1892{
1893 AssertPtr(pVCpu);
1894 AssertPtr(pCpu);
1895
1896 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1897
1898 Assert(pCpu->idCpu != NIL_RTCPUID);
1899 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1900 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1901 pVCpu->hm.s.fForceTLBFlush = false;
1902 return;
1903}
1904
1905
1906/**
1907 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1908 *
1909 * @param pVCpu The cross context virtual CPU structure.
1910 * @param pCpu Pointer to the global HM CPU struct.
1911 *
1912 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1913 * nomenclature. The reason is, to avoid confusion in compare statements
1914 * since the host-CPU copies are named "ASID".
1915 *
1916 * @remarks Called with interrupts disabled.
1917 */
1918static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1919{
1920#ifdef VBOX_WITH_STATISTICS
1921 bool fTlbFlushed = false;
1922# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1923# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1924 if (!fTlbFlushed) \
1925 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1926 } while (0)
1927#else
1928# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1929# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1930#endif
1931
1932 AssertPtr(pCpu);
1933 AssertPtr(pVCpu);
1934 Assert(pCpu->idCpu != NIL_RTCPUID);
1935
1936 PVM pVM = pVCpu->CTX_SUFF(pVM);
1937 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1938 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1939 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1940
1941 /*
1942 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1943 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1944 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1945 * cannot reuse the current ASID anymore.
1946 */
1947 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1948 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1949 {
1950 ++pCpu->uCurrentAsid;
1951 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1952 {
1953 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1954 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1955 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1956 }
1957
1958 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1959 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1960 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1961
1962 /*
1963 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1964 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1965 */
1966 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1967 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1968 HMVMX_SET_TAGGED_TLB_FLUSHED();
1969 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1970 }
1971 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1972 {
1973 /*
1974 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1975 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1976 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1977 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1978 * mappings, see @bugref{6568}.
1979 *
1980 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1981 */
1982 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1983 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1984 HMVMX_SET_TAGGED_TLB_FLUSHED();
1985 }
1986
1987 pVCpu->hm.s.fForceTLBFlush = false;
1988 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1989
1990 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1991 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1992 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1993 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1994 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1995 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1996 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1997 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1998 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1999
2000 /* Update VMCS with the VPID. */
2001 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2002 AssertRC(rc);
2003
2004#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2005}
2006
2007
2008/**
2009 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2010 *
2011 * @returns VBox status code.
2012 * @param pVCpu The cross context virtual CPU structure.
2013 * @param pCpu Pointer to the global HM CPU struct.
2014 *
2015 * @remarks Called with interrupts disabled.
2016 */
2017static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2018{
2019 AssertPtr(pVCpu);
2020 AssertPtr(pCpu);
2021 Assert(pCpu->idCpu != NIL_RTCPUID);
2022 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2023 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2024
2025 /*
2026 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2027 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2028 */
2029 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2030 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2031 {
2032 pVCpu->hm.s.fForceTLBFlush = true;
2033 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034 }
2035
2036 /* Check for explicit TLB flushes. */
2037 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2038 {
2039 pVCpu->hm.s.fForceTLBFlush = true;
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2041 }
2042
2043 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2044 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2045
2046 if (pVCpu->hm.s.fForceTLBFlush)
2047 {
2048 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2049 pVCpu->hm.s.fForceTLBFlush = false;
2050 }
2051}
2052
2053
2054/**
2055 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2056 *
2057 * @returns VBox status code.
2058 * @param pVCpu The cross context virtual CPU structure.
2059 * @param pCpu Pointer to the global HM CPU struct.
2060 *
2061 * @remarks Called with interrupts disabled.
2062 */
2063static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2064{
2065 AssertPtr(pVCpu);
2066 AssertPtr(pCpu);
2067 Assert(pCpu->idCpu != NIL_RTCPUID);
2068 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2069 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2070
2071 /*
2072 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2073 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2074 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2075 * cannot reuse the current ASID anymore.
2076 */
2077 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2078 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2079 {
2080 pVCpu->hm.s.fForceTLBFlush = true;
2081 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2082 }
2083
2084 /* Check for explicit TLB flushes. */
2085 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2086 {
2087 /*
2088 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2089 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2090 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2091 * include fExplicitFlush's too) - an obscure corner case.
2092 */
2093 pVCpu->hm.s.fForceTLBFlush = true;
2094 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2095 }
2096
2097 PVM pVM = pVCpu->CTX_SUFF(pVM);
2098 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2099 if (pVCpu->hm.s.fForceTLBFlush)
2100 {
2101 ++pCpu->uCurrentAsid;
2102 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2103 {
2104 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2105 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2106 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2107 }
2108
2109 pVCpu->hm.s.fForceTLBFlush = false;
2110 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2111 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2112 if (pCpu->fFlushAsidBeforeUse)
2113 {
2114 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2115 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2116 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2117 {
2118 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2119 pCpu->fFlushAsidBeforeUse = false;
2120 }
2121 else
2122 {
2123 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2124 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2125 }
2126 }
2127 }
2128
2129 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2130 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2131 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2132 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2133 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2134 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2135 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2136
2137 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2138 AssertRC(rc);
2139}
2140
2141
2142/**
2143 * Flushes the guest TLB entry based on CPU capabilities.
2144 *
2145 * @param pVCpu The cross context virtual CPU structure.
2146 * @param pCpu Pointer to the global HM CPU struct.
2147 */
2148DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2149{
2150#ifdef HMVMX_ALWAYS_FLUSH_TLB
2151 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2152#endif
2153 PVM pVM = pVCpu->CTX_SUFF(pVM);
2154 switch (pVM->hm.s.vmx.enmTlbFlushType)
2155 {
2156 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2157 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2158 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2159 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2160 default:
2161 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2162 break;
2163 }
2164 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2165}
2166
2167
2168/**
2169 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2170 * TLB entries from the host TLB before VM-entry.
2171 *
2172 * @returns VBox status code.
2173 * @param pVM The cross context VM structure.
2174 */
2175static int hmR0VmxSetupTaggedTlb(PVM pVM)
2176{
2177 /*
2178 * Determine optimal flush type for Nested Paging.
2179 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2180 * guest execution (see hmR3InitFinalizeR0()).
2181 */
2182 if (pVM->hm.s.fNestedPaging)
2183 {
2184 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2185 {
2186 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2187 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2188 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2189 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2190 else
2191 {
2192 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2193 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2194 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2195 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2196 }
2197
2198 /* Make sure the write-back cacheable memory type for EPT is supported. */
2199 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2200 {
2201 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2202 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2203 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2204 }
2205
2206 /* EPT requires a page-walk length of 4. */
2207 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2208 {
2209 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2210 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2211 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2212 }
2213 }
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2217 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221 }
2222
2223 /*
2224 * Determine optimal flush type for VPID.
2225 */
2226 if (pVM->hm.s.vmx.fVpid)
2227 {
2228 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2229 {
2230 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2231 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2232 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2233 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2234 else
2235 {
2236 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2238 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2239 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2240 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2241 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2242 pVM->hm.s.vmx.fVpid = false;
2243 }
2244 }
2245 else
2246 {
2247 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2248 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2249 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2250 pVM->hm.s.vmx.fVpid = false;
2251 }
2252 }
2253
2254 /*
2255 * Setup the handler for flushing tagged-TLBs.
2256 */
2257 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2258 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2259 else if (pVM->hm.s.fNestedPaging)
2260 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2261 else if (pVM->hm.s.vmx.fVpid)
2262 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2263 else
2264 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2265 return VINF_SUCCESS;
2266}
2267
2268
2269/**
2270 * Sets up pin-based VM-execution controls in the VMCS.
2271 *
2272 * @returns VBox status code.
2273 * @param pVCpu The cross context virtual CPU structure.
2274 *
2275 * @remarks We don't really care about optimizing vmwrites here as it's done only
2276 * once per VM and hence we don't care about VMCS-field cache comparisons.
2277 */
2278static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2279{
2280 PVM pVM = pVCpu->CTX_SUFF(pVM);
2281 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2282 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2283
2284 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2285 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2286
2287 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2288 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2289
2290 /* Enable the VMX preemption timer. */
2291 if (pVM->hm.s.vmx.fUsePreemptTimer)
2292 {
2293 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2294 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2295 }
2296
2297#if 0
2298 /* Enable posted-interrupt processing. */
2299 if (pVM->hm.s.fPostedIntrs)
2300 {
2301 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2302 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2303 fVal |= VMX_PIN_CTL_POSTED_INT;
2304 }
2305#endif
2306
2307 if ((fVal & fZap) != fVal)
2308 {
2309 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2310 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2311 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2312 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2313 }
2314
2315 /* Commit it to the VMCS and update our cache. */
2316 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2317 AssertRCReturn(rc, rc);
2318 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2319
2320 return VINF_SUCCESS;
2321}
2322
2323
2324/**
2325 * Sets up secondary processor-based VM-execution controls in the VMCS.
2326 *
2327 * @returns VBox status code.
2328 * @param pVCpu The cross context virtual CPU structure.
2329 *
2330 * @remarks We don't really care about optimizing vmwrites here as it's done only
2331 * once per VM and hence we don't care about VMCS-field cache comparisons.
2332 */
2333static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2334{
2335 PVM pVM = pVCpu->CTX_SUFF(pVM);
2336 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2337 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2338
2339 /* WBINVD causes a VM-exit. */
2340 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2341 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2342
2343 /* Enable EPT (aka nested-paging). */
2344 if (pVM->hm.s.fNestedPaging)
2345 fVal |= VMX_PROC_CTLS2_EPT;
2346
2347 /*
2348 * Enable the INVPCID instruction if supported by the hardware and we expose
2349 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2350 */
2351 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2352 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2353 fVal |= VMX_PROC_CTLS2_INVPCID;
2354
2355 /* Enable VPID. */
2356 if (pVM->hm.s.vmx.fVpid)
2357 fVal |= VMX_PROC_CTLS2_VPID;
2358
2359 /* Enable Unrestricted guest execution. */
2360 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2361 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2362
2363#if 0
2364 if (pVM->hm.s.fVirtApicRegs)
2365 {
2366 /* Enable APIC-register virtualization. */
2367 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2368 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2369
2370 /* Enable virtual-interrupt delivery. */
2371 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2372 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2373 }
2374#endif
2375
2376 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2377 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2378 * done dynamically. */
2379 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2380 {
2381 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2382 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2383 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2384 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2385 AssertRCReturn(rc, rc);
2386 }
2387
2388 /* Enable RDTSCP. */
2389 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2390 fVal |= VMX_PROC_CTLS2_RDTSCP;
2391
2392 /* Enable Pause-Loop exiting. */
2393 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2394 && pVM->hm.s.vmx.cPleGapTicks
2395 && pVM->hm.s.vmx.cPleWindowTicks)
2396 {
2397 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2398
2399 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2400 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2401 AssertRCReturn(rc, rc);
2402 }
2403
2404 if ((fVal & fZap) != fVal)
2405 {
2406 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2407 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2408 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2409 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2410 }
2411
2412 /* Commit it to the VMCS and update our cache. */
2413 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2414 AssertRCReturn(rc, rc);
2415 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2416
2417 return VINF_SUCCESS;
2418}
2419
2420
2421/**
2422 * Sets up processor-based VM-execution controls in the VMCS.
2423 *
2424 * @returns VBox status code.
2425 * @param pVCpu The cross context virtual CPU structure.
2426 *
2427 * @remarks We don't really care about optimizing vmwrites here as it's done only
2428 * once per VM and hence we don't care about VMCS-field cache comparisons.
2429 */
2430static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2431{
2432 PVM pVM = pVCpu->CTX_SUFF(pVM);
2433 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2434 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2435
2436 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2437 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2438 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2439 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2440 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2441 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2442 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2443
2444 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2445 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2446 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2447 {
2448 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2449 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2450 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2451 }
2452
2453 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2454 if (!pVM->hm.s.fNestedPaging)
2455 {
2456 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2457 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2458 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2459 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2460 }
2461
2462 /* Use TPR shadowing if supported by the CPU. */
2463 if ( PDMHasApic(pVM)
2464 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2465 {
2466 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2467 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2468 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2469 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2470 AssertRCReturn(rc, rc);
2471
2472 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2473 /* CR8 writes cause a VM-exit based on TPR threshold. */
2474 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2475 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2476 }
2477 else
2478 {
2479 /*
2480 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2481 * Set this control only for 64-bit guests.
2482 */
2483 if (pVM->hm.s.fAllow64BitGuests)
2484 {
2485 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2486 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2487 }
2488 }
2489
2490 /* Use MSR-bitmaps if supported by the CPU. */
2491 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2492 {
2493 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2494
2495 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2496 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2497 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2498 AssertRCReturn(rc, rc);
2499
2500 /*
2501 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2502 * automatically using dedicated fields in the VMCS.
2503 */
2504 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2505 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2507 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2508 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2509#if HC_ARCH_BITS == 64
2510 /*
2511 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2512 */
2513 if (pVM->hm.s.fAllow64BitGuests)
2514 {
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2519 }
2520#endif
2521 /*
2522 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2523 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2524 */
2525 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2526 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2527
2528 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2529 }
2530
2531 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2532 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2533 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2534
2535 if ((fVal & fZap) != fVal)
2536 {
2537 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2538 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2539 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2540 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2541 }
2542
2543 /* Commit it to the VMCS and update our cache. */
2544 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2545 AssertRCReturn(rc, rc);
2546 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2547
2548 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2549 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2550 return hmR0VmxSetupProcCtls2(pVCpu);
2551
2552 /* Sanity check, should not really happen. */
2553 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2554 {
2555 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2561 return VINF_SUCCESS;
2562}
2563
2564
2565/**
2566 * Sets up miscellaneous (everything other than Pin & Processor-based
2567 * VM-execution) control fields in the VMCS.
2568 *
2569 * @returns VBox status code.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2573{
2574 AssertPtr(pVCpu);
2575
2576 int rc = VERR_GENERAL_FAILURE;
2577
2578 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2579#if 0
2580 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2581 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2582 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2583
2584 /*
2585 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2586 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2587 * We thus use the exception bitmap to control it rather than use both.
2588 */
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2590 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2591
2592 /* All IO & IOIO instructions cause VM-exits. */
2593 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2594 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2595
2596 /* Initialize the MSR-bitmap area. */
2597 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2598 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2599 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2600 AssertRCReturn(rc, rc);
2601#endif
2602
2603 /* Setup MSR auto-load/store area. */
2604 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2605 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2606 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2607 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 AssertRCReturn(rc, rc);
2609
2610 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2611 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2612 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2613 AssertRCReturn(rc, rc);
2614
2615 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2616 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2617 AssertRCReturn(rc, rc);
2618
2619 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2620#if 0
2621 /* Setup debug controls */
2622 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2623 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2624 AssertRCReturn(rc, rc);
2625#endif
2626
2627 return rc;
2628}
2629
2630
2631/**
2632 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2633 *
2634 * We shall setup those exception intercepts that don't change during the
2635 * lifetime of the VM here. The rest are done dynamically while loading the
2636 * guest state.
2637 *
2638 * @returns VBox status code.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2642{
2643 AssertPtr(pVCpu);
2644
2645 uint32_t uXcptBitmap;
2646
2647 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2648 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2649
2650 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2651 and writes, and because recursive #DBs can cause the CPU hang, we must always
2652 intercept #DB. */
2653 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2654
2655 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2656 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2657 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2658
2659 /* Commit it to the VMCS. */
2660 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2661 AssertRCReturn(rc, rc);
2662
2663 /* Update our cache of the exception bitmap. */
2664 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2665 return VINF_SUCCESS;
2666}
2667
2668
2669/**
2670 * Does per-VM VT-x initialization.
2671 *
2672 * @returns VBox status code.
2673 * @param pVM The cross context VM structure.
2674 */
2675VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2676{
2677 LogFlowFunc(("pVM=%p\n", pVM));
2678
2679 int rc = hmR0VmxStructsAlloc(pVM);
2680 if (RT_FAILURE(rc))
2681 {
2682 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2683 return rc;
2684 }
2685
2686 return VINF_SUCCESS;
2687}
2688
2689
2690/**
2691 * Does per-VM VT-x termination.
2692 *
2693 * @returns VBox status code.
2694 * @param pVM The cross context VM structure.
2695 */
2696VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2697{
2698 LogFlowFunc(("pVM=%p\n", pVM));
2699
2700#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2701 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2702 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2703#endif
2704 hmR0VmxStructsFree(pVM);
2705 return VINF_SUCCESS;
2706}
2707
2708
2709/**
2710 * Sets up the VM for execution under VT-x.
2711 * This function is only called once per-VM during initialization.
2712 *
2713 * @returns VBox status code.
2714 * @param pVM The cross context VM structure.
2715 */
2716VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2717{
2718 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2719 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2720
2721 LogFlowFunc(("pVM=%p\n", pVM));
2722
2723 /*
2724 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2725 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2726 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2727 */
2728 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2729 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2730 || !pVM->hm.s.vmx.pRealModeTSS))
2731 {
2732 LogRelFunc(("Invalid real-on-v86 state.\n"));
2733 return VERR_INTERNAL_ERROR;
2734 }
2735
2736 /* Initialize these always, see hmR3InitFinalizeR0().*/
2737 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2738 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2739
2740 /* Setup the tagged-TLB flush handlers. */
2741 int rc = hmR0VmxSetupTaggedTlb(pVM);
2742 if (RT_FAILURE(rc))
2743 {
2744 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2745 return rc;
2746 }
2747
2748 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2749 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2750#if HC_ARCH_BITS == 64
2751 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2752 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2753 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2754 {
2755 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2756 }
2757#endif
2758
2759 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2760 RTCCUINTREG uHostCR4 = ASMGetCR4();
2761 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2762 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2763
2764 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2765 {
2766 PVMCPU pVCpu = &pVM->aCpus[i];
2767 AssertPtr(pVCpu);
2768 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2769
2770 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2771 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2772
2773 /* Set revision dword at the beginning of the VMCS structure. */
2774 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2775
2776 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2777 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2778 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2779 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2780
2781 /* Load this VMCS as the current VMCS. */
2782 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2783 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2784 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2785
2786 rc = hmR0VmxSetupPinCtls(pVCpu);
2787 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2788 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2789
2790 rc = hmR0VmxSetupProcCtls(pVCpu);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2792 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2793
2794 rc = hmR0VmxSetupMiscCtls(pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2796 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2797
2798 rc = hmR0VmxInitXcptBitmap(pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2800 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2801
2802#if HC_ARCH_BITS == 32
2803 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2805 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2806#endif
2807
2808 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2809 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2810 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2811 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2812
2813 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2814
2815 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2816 }
2817
2818 return VINF_SUCCESS;
2819}
2820
2821
2822/**
2823 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2824 * the VMCS.
2825 *
2826 * @returns VBox status code.
2827 */
2828static int hmR0VmxExportHostControlRegs(void)
2829{
2830 RTCCUINTREG uReg = ASMGetCR0();
2831 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2832 AssertRCReturn(rc, rc);
2833
2834 uReg = ASMGetCR3();
2835 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2836 AssertRCReturn(rc, rc);
2837
2838 uReg = ASMGetCR4();
2839 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2840 AssertRCReturn(rc, rc);
2841 return rc;
2842}
2843
2844
2845/**
2846 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2847 * the host-state area in the VMCS.
2848 *
2849 * @returns VBox status code.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2853{
2854#if HC_ARCH_BITS == 64
2855/**
2856 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2857 * requirements. See hmR0VmxExportHostSegmentRegs().
2858 */
2859# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2860 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2861 { \
2862 bool fValidSelector = true; \
2863 if ((selValue) & X86_SEL_LDT) \
2864 { \
2865 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2866 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2867 } \
2868 if (fValidSelector) \
2869 { \
2870 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2871 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2872 } \
2873 (selValue) = 0; \
2874 }
2875
2876 /*
2877 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2878 * should -not- save the messed up state without restoring the original host-state,
2879 * see @bugref{7240}.
2880 *
2881 * This apparently can happen (most likely the FPU changes), deal with it rather than
2882 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2883 */
2884 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2885 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2886 {
2887 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2888 pVCpu->idCpu));
2889 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2890 }
2891 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2892#else
2893 RT_NOREF(pVCpu);
2894#endif
2895
2896 /*
2897 * Host DS, ES, FS and GS segment registers.
2898 */
2899#if HC_ARCH_BITS == 64
2900 RTSEL uSelDS = ASMGetDS();
2901 RTSEL uSelES = ASMGetES();
2902 RTSEL uSelFS = ASMGetFS();
2903 RTSEL uSelGS = ASMGetGS();
2904#else
2905 RTSEL uSelDS = 0;
2906 RTSEL uSelES = 0;
2907 RTSEL uSelFS = 0;
2908 RTSEL uSelGS = 0;
2909#endif
2910
2911 /*
2912 * Host CS and SS segment registers.
2913 */
2914 RTSEL uSelCS = ASMGetCS();
2915 RTSEL uSelSS = ASMGetSS();
2916
2917 /*
2918 * Host TR segment register.
2919 */
2920 RTSEL uSelTR = ASMGetTR();
2921
2922#if HC_ARCH_BITS == 64
2923 /*
2924 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2925 * gain VM-entry and restore them before we get preempted.
2926 *
2927 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2928 */
2929 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2930 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2931 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2932 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2933# undef VMXLOCAL_ADJUST_HOST_SEG
2934#endif
2935
2936 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2937 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2938 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2939 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2940 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2941 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2942 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2943 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2944 Assert(uSelCS);
2945 Assert(uSelTR);
2946
2947 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2948#if 0
2949 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2950 Assert(uSelSS != 0);
2951#endif
2952
2953 /* Write these host selector fields into the host-state area in the VMCS. */
2954 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2955 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2956#if HC_ARCH_BITS == 64
2957 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2958 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2959 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2960 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2961#else
2962 NOREF(uSelDS);
2963 NOREF(uSelES);
2964 NOREF(uSelFS);
2965 NOREF(uSelGS);
2966#endif
2967 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2968 AssertRCReturn(rc, rc);
2969
2970 /*
2971 * Host GDTR and IDTR.
2972 */
2973 RTGDTR Gdtr;
2974 RTIDTR Idtr;
2975 RT_ZERO(Gdtr);
2976 RT_ZERO(Idtr);
2977 ASMGetGDTR(&Gdtr);
2978 ASMGetIDTR(&Idtr);
2979 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2980 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2981 AssertRCReturn(rc, rc);
2982
2983#if HC_ARCH_BITS == 64
2984 /*
2985 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2986 * them to the maximum limit (0xffff) on every VM-exit.
2987 */
2988 if (Gdtr.cbGdt != 0xffff)
2989 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2990
2991 /*
2992 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2993 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
2994 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
2995 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
2996 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
2997 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
2998 * at 0xffff on hosts where we are sure it won't cause trouble.
2999 */
3000# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3001 if (Idtr.cbIdt < 0x0fff)
3002# else
3003 if (Idtr.cbIdt != 0xffff)
3004# endif
3005 {
3006 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3007 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3008 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3009 }
3010#endif
3011
3012 /*
3013 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3014 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3015 * RPL should be too in most cases.
3016 */
3017 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3018 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3019
3020 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3021#if HC_ARCH_BITS == 64
3022 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3023
3024 /*
3025 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3026 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3027 * restoration if the host has something else. Task switching is not supported in 64-bit
3028 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3029 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3030 *
3031 * [1] See Intel spec. 3.5 "System Descriptor Types".
3032 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3033 */
3034 PVM pVM = pVCpu->CTX_SUFF(pVM);
3035 Assert(pDesc->System.u4Type == 11);
3036 if ( pDesc->System.u16LimitLow != 0x67
3037 || pDesc->System.u4LimitHigh)
3038 {
3039 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3040 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3041 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3042 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3043 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3044 }
3045
3046 /*
3047 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3048 */
3049 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3050 {
3051 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3052 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3053 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3054 {
3055 /* The GDT is read-only but the writable GDT is available. */
3056 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3057 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3058 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3059 AssertRCReturn(rc, rc);
3060 }
3061 }
3062#else
3063 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3064#endif
3065 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3066 AssertRCReturn(rc, rc);
3067
3068 /*
3069 * Host FS base and GS base.
3070 */
3071#if HC_ARCH_BITS == 64
3072 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3073 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3074 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3075 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3076 AssertRCReturn(rc, rc);
3077
3078 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3079 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3080 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3081 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3082 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3083#endif
3084 return VINF_SUCCESS;
3085}
3086
3087
3088/**
3089 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3090 * host-state area of the VMCS.
3091 *
3092 * Theses MSRs will be automatically restored on the host after every successful
3093 * VM-exit.
3094 *
3095 * @returns VBox status code.
3096 * @param pVCpu The cross context virtual CPU structure.
3097 *
3098 * @remarks No-long-jump zone!!!
3099 */
3100static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3101{
3102 AssertPtr(pVCpu);
3103 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3104
3105 /*
3106 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3107 * rather than swapping them on every VM-entry.
3108 */
3109 hmR0VmxLazySaveHostMsrs(pVCpu);
3110
3111 /*
3112 * Host Sysenter MSRs.
3113 */
3114 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3115#if HC_ARCH_BITS == 32
3116 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3117 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3118#else
3119 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3120 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3121#endif
3122 AssertRCReturn(rc, rc);
3123
3124 /*
3125 * Host EFER MSR.
3126 *
3127 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3128 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3129 */
3130 PVM pVM = pVCpu->CTX_SUFF(pVM);
3131 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3132 {
3133 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3134 AssertRCReturn(rc, rc);
3135 }
3136
3137 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3138
3139 return VINF_SUCCESS;
3140}
3141
3142
3143/**
3144 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3145 *
3146 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3147 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3148 * hmR0VMxExportGuestEntryCtls().
3149 *
3150 * @returns true if we need to load guest EFER, false otherwise.
3151 * @param pVCpu The cross context virtual CPU structure.
3152 *
3153 * @remarks Requires EFER, CR4.
3154 * @remarks No-long-jump zone!!!
3155 */
3156static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3157{
3158#ifdef HMVMX_ALWAYS_SWAP_EFER
3159 return true;
3160#endif
3161
3162 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3163#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3164 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3165 if (CPUMIsGuestInLongModeEx(pCtx))
3166 return false;
3167#endif
3168
3169 PVM pVM = pVCpu->CTX_SUFF(pVM);
3170 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3171 uint64_t const u64GuestEfer = pCtx->msrEFER;
3172
3173 /*
3174 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3175 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3176 */
3177 if ( CPUMIsGuestInLongModeEx(pCtx)
3178 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3179 {
3180 return true;
3181 }
3182
3183 /*
3184 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3185 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3186 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3187 */
3188 if ( (pCtx->cr4 & X86_CR4_PAE)
3189 && (pCtx->cr0 & X86_CR0_PG)
3190 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3191 {
3192 /* Assert that host is NX capable. */
3193 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3194 return true;
3195 }
3196
3197 return false;
3198}
3199
3200
3201/**
3202 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3203 *
3204 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3205 * see Intel spec. 24.8.1 "VM-entry controls".
3206 *
3207 * @returns VBox status code.
3208 * @param pVCpu The cross context virtual CPU structure.
3209 *
3210 * @remarks Requires EFER.
3211 * @remarks No-long-jump zone!!!
3212 */
3213static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3214{
3215 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3216 {
3217 PVM pVM = pVCpu->CTX_SUFF(pVM);
3218 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3219 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3220
3221 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3222 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3223
3224 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3225 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3226 {
3227 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3228 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3229 }
3230 else
3231 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3232
3233 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3234 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3235 && hmR0VmxShouldSwapEferMsr(pVCpu))
3236 {
3237 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3238 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3239 }
3240
3241 /*
3242 * The following should -not- be set (since we're not in SMM mode):
3243 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3244 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3245 */
3246
3247 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3248 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3249
3250 if ((fVal & fZap) != fVal)
3251 {
3252 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3253 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3254 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3255 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3256 }
3257
3258 /* Commit it to the VMCS and update our cache. */
3259 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3260 {
3261 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3262 AssertRCReturn(rc, rc);
3263 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3264 }
3265
3266 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3267 }
3268 return VINF_SUCCESS;
3269}
3270
3271
3272/**
3273 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3274 *
3275 * @returns VBox status code.
3276 * @param pVCpu The cross context virtual CPU structure.
3277 *
3278 * @remarks Requires EFER.
3279 */
3280static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3281{
3282 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3283 {
3284 PVM pVM = pVCpu->CTX_SUFF(pVM);
3285 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3286 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3287
3288 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3289 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3290
3291 /*
3292 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3293 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3294 * hmR0VmxExportHostMsrs().
3295 */
3296#if HC_ARCH_BITS == 64
3297 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3298 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3299#else
3300 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3301 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3302 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3303 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3304 {
3305 /* The switcher returns to long mode, EFER is managed by the switcher. */
3306 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3307 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3308 }
3309 else
3310 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3311#endif
3312
3313 /* If the newer VMCS fields for managing EFER exists, use it. */
3314 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3315 && hmR0VmxShouldSwapEferMsr(pVCpu))
3316 {
3317 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3318 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3319 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3320 }
3321
3322 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3323 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3324
3325 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3326 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3327 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3328
3329 /* Enable saving of the VMX preemption timer value on VM-exit. */
3330 if ( pVM->hm.s.vmx.fUsePreemptTimer
3331 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_VMX_PREEMPT_TIMER))
3332 fVal |= VMX_EXIT_CTLS_SAVE_VMX_PREEMPT_TIMER;
3333
3334 if ((fVal & fZap) != fVal)
3335 {
3336 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3337 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3338 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3339 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3340 }
3341
3342 /* Commit it to the VMCS and update our cache. */
3343 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3344 {
3345 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3346 AssertRCReturn(rc, rc);
3347 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3348 }
3349
3350 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3351 }
3352 return VINF_SUCCESS;
3353}
3354
3355
3356/**
3357 * Sets the TPR threshold in the VMCS.
3358 *
3359 * @returns VBox status code.
3360 * @param pVCpu The cross context virtual CPU structure.
3361 * @param u32TprThreshold The TPR threshold (task-priority class only).
3362 */
3363DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3364{
3365 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3366 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3367 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3368}
3369
3370
3371/**
3372 * Exports the guest APIC TPR state into the VMCS.
3373 *
3374 * @returns VBox status code.
3375 * @param pVCpu The cross context virtual CPU structure.
3376 *
3377 * @remarks No-long-jump zone!!!
3378 */
3379static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3380{
3381 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3382 {
3383 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3384
3385 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3386 && APICIsEnabled(pVCpu))
3387 {
3388 /*
3389 * Setup TPR shadowing.
3390 */
3391 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3392 {
3393 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3394
3395 bool fPendingIntr = false;
3396 uint8_t u8Tpr = 0;
3397 uint8_t u8PendingIntr = 0;
3398 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3399 AssertRCReturn(rc, rc);
3400
3401 /*
3402 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3403 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3404 * priority of the pending interrupt so we can deliver the interrupt. If there
3405 * are no interrupts pending, set threshold to 0 to not cause any
3406 * TPR-below-threshold VM-exits.
3407 */
3408 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3409 uint32_t u32TprThreshold = 0;
3410 if (fPendingIntr)
3411 {
3412 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3413 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3414 const uint8_t u8TprPriority = u8Tpr >> 4;
3415 if (u8PendingPriority <= u8TprPriority)
3416 u32TprThreshold = u8PendingPriority;
3417 }
3418
3419 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3420 AssertRCReturn(rc, rc);
3421 }
3422 }
3423 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3424 }
3425 return VINF_SUCCESS;
3426}
3427
3428
3429/**
3430 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3431 *
3432 * @returns Guest's interruptibility-state.
3433 * @param pVCpu The cross context virtual CPU structure.
3434 *
3435 * @remarks No-long-jump zone!!!
3436 */
3437static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3438{
3439 /*
3440 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3441 */
3442 uint32_t fIntrState = 0;
3443 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3444 {
3445 /* If inhibition is active, RIP & RFLAGS should've been accessed
3446 (i.e. read previously from the VMCS or from ring-3). */
3447 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3448#ifdef VBOX_STRICT
3449 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3450 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3451#endif
3452 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3453 {
3454 if (pCtx->eflags.Bits.u1IF)
3455 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3456 else
3457 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3458 }
3459 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3460 {
3461 /*
3462 * We can clear the inhibit force flag as even if we go back to the recompiler
3463 * without executing guest code in VT-x, the flag's condition to be cleared is
3464 * met and thus the cleared state is correct.
3465 */
3466 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3467 }
3468 }
3469
3470 /*
3471 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3472 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3473 * setting this would block host-NMIs and IRET will not clear the blocking.
3474 *
3475 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3476 */
3477 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3478 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3479 {
3480 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3481 }
3482
3483 return fIntrState;
3484}
3485
3486
3487/**
3488 * Exports the guest's interruptibility-state into the guest-state area in the
3489 * VMCS.
3490 *
3491 * @returns VBox status code.
3492 * @param pVCpu The cross context virtual CPU structure.
3493 * @param fIntrState The interruptibility-state to set.
3494 */
3495static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3496{
3497 NOREF(pVCpu);
3498 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3499 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3500 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3501}
3502
3503
3504/**
3505 * Exports the exception intercepts required for guest execution in the VMCS.
3506 *
3507 * @returns VBox status code.
3508 * @param pVCpu The cross context virtual CPU structure.
3509 *
3510 * @remarks No-long-jump zone!!!
3511 */
3512static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3513{
3514 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3515 {
3516 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3517
3518 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3519 if (pVCpu->hm.s.fGIMTrapXcptUD)
3520 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3521#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3522 else
3523 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3524#endif
3525
3526 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3527 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3528
3529 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3530 {
3531 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3532 AssertRCReturn(rc, rc);
3533 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3534 }
3535
3536 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3537 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3538 }
3539 return VINF_SUCCESS;
3540}
3541
3542
3543/**
3544 * Exports the guest's RIP into the guest-state area in the VMCS.
3545 *
3546 * @returns VBox status code.
3547 * @param pVCpu The cross context virtual CPU structure.
3548 *
3549 * @remarks No-long-jump zone!!!
3550 */
3551static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3552{
3553 int rc = VINF_SUCCESS;
3554 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3555 {
3556 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3557
3558 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3559 AssertRCReturn(rc, rc);
3560
3561 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3562 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3563 }
3564 return rc;
3565}
3566
3567
3568/**
3569 * Exports the guest's RSP into the guest-state area in the VMCS.
3570 *
3571 * @returns VBox status code.
3572 * @param pVCpu The cross context virtual CPU structure.
3573 *
3574 * @remarks No-long-jump zone!!!
3575 */
3576static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3577{
3578 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3579 {
3580 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3581
3582 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3583 AssertRCReturn(rc, rc);
3584
3585 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3586 }
3587 return VINF_SUCCESS;
3588}
3589
3590
3591/**
3592 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3593 *
3594 * @returns VBox status code.
3595 * @param pVCpu The cross context virtual CPU structure.
3596 *
3597 * @remarks No-long-jump zone!!!
3598 */
3599static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3600{
3601 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3602 {
3603 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3604
3605 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3606 Let us assert it as such and use 32-bit VMWRITE. */
3607 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3608 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3609 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3610 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3611
3612 /*
3613 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3614 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3615 * can run the real-mode guest code under Virtual 8086 mode.
3616 */
3617 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3618 {
3619 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3620 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3621 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3622 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3623 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3624 }
3625
3626 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3627 AssertRCReturn(rc, rc);
3628
3629 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3630 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3631 }
3632 return VINF_SUCCESS;
3633}
3634
3635
3636/**
3637 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3638 *
3639 * The guest FPU state is always pre-loaded hence we don't need to bother about
3640 * sharing FPU related CR0 bits between the guest and host.
3641 *
3642 * @returns VBox status code.
3643 * @param pVCpu The cross context virtual CPU structure.
3644 *
3645 * @remarks No-long-jump zone!!!
3646 */
3647static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3648{
3649 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3650 {
3651 PVM pVM = pVCpu->CTX_SUFF(pVM);
3652 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3653 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3654
3655 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3656 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3657
3658 /*
3659 * Setup VT-x's view of the guest CR0.
3660 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3661 */
3662 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3663 if (pVM->hm.s.fNestedPaging)
3664 {
3665 if (CPUMIsGuestPagingEnabled(pVCpu))
3666 {
3667 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3668 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3669 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3670 }
3671 else
3672 {
3673 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3674 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3675 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3676 }
3677
3678 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3679 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3680 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3681 }
3682 else
3683 {
3684 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3685 u32GuestCr0 |= X86_CR0_WP;
3686 }
3687
3688 /*
3689 * Guest FPU bits.
3690 *
3691 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3692 * using CR0.TS.
3693 *
3694 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3695 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3696 */
3697 u32GuestCr0 |= X86_CR0_NE;
3698
3699 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3700 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3701
3702 /*
3703 * Update exception intercepts.
3704 */
3705 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3706 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3707 {
3708 Assert(PDMVmmDevHeapIsEnabled(pVM));
3709 Assert(pVM->hm.s.vmx.pRealModeTSS);
3710 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3711 }
3712 else
3713 {
3714 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3715 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3716 if (fInterceptMF)
3717 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3718 }
3719
3720 /* Additional intercepts for debugging, define these yourself explicitly. */
3721#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3722 uXcptBitmap |= 0
3723 | RT_BIT(X86_XCPT_BP)
3724 | RT_BIT(X86_XCPT_DE)
3725 | RT_BIT(X86_XCPT_NM)
3726 | RT_BIT(X86_XCPT_TS)
3727 | RT_BIT(X86_XCPT_UD)
3728 | RT_BIT(X86_XCPT_NP)
3729 | RT_BIT(X86_XCPT_SS)
3730 | RT_BIT(X86_XCPT_GP)
3731 | RT_BIT(X86_XCPT_PF)
3732 | RT_BIT(X86_XCPT_MF)
3733 ;
3734#elif defined(HMVMX_ALWAYS_TRAP_PF)
3735 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3736#endif
3737 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3738
3739 /*
3740 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3741 */
3742 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3743 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3744 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3745 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3746 else
3747 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3748
3749 u32GuestCr0 |= fSetCr0;
3750 u32GuestCr0 &= fZapCr0;
3751 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3752
3753 /*
3754 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3755 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3756 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3757 */
3758 uint32_t u32Cr0Mask = X86_CR0_PE
3759 | X86_CR0_NE
3760 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3761 | X86_CR0_PG
3762 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3763 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3764 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3765
3766 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3767 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3768 * and @bugref{6944}. */
3769#if 0
3770 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3771 u32Cr0Mask &= ~X86_CR0_PE;
3772#endif
3773 /*
3774 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3775 */
3776 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3777 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3778 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3779 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3780 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3781 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3782 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3783 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3784 AssertRCReturn(rc, rc);
3785
3786 /* Update our caches. */
3787 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3788 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3789 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3790
3791 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3792
3793 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3794 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3795 }
3796
3797 return VINF_SUCCESS;
3798}
3799
3800
3801/**
3802 * Exports the guest control registers (CR3, CR4) into the guest-state area
3803 * in the VMCS.
3804 *
3805 * @returns VBox strict status code.
3806 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3807 * without unrestricted guest access and the VMMDev is not presently
3808 * mapped (e.g. EFI32).
3809 *
3810 * @param pVCpu The cross context virtual CPU structure.
3811 *
3812 * @remarks No-long-jump zone!!!
3813 */
3814static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3815{
3816 int rc = VINF_SUCCESS;
3817 PVM pVM = pVCpu->CTX_SUFF(pVM);
3818
3819 /*
3820 * Guest CR2.
3821 * It's always loaded in the assembler code. Nothing to do here.
3822 */
3823
3824 /*
3825 * Guest CR3.
3826 */
3827 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3828 {
3829 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3830
3831 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3832 if (pVM->hm.s.fNestedPaging)
3833 {
3834 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3835
3836 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3837 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3838 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3839 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3840
3841 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3842 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3843 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3844
3845 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3846 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3847 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3848 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3849 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3850 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3851 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3852
3853 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3854 AssertRCReturn(rc, rc);
3855
3856 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3857 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3858 || CPUMIsGuestPagingEnabledEx(pCtx))
3859 {
3860 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3861 if (CPUMIsGuestInPAEModeEx(pCtx))
3862 {
3863 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3864 AssertRCReturn(rc, rc);
3865 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3866 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3867 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3868 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3869 AssertRCReturn(rc, rc);
3870 }
3871
3872 /*
3873 * The guest's view of its CR3 is unblemished with Nested Paging when the
3874 * guest is using paging or we have unrestricted guest execution to handle
3875 * the guest when it's not using paging.
3876 */
3877 GCPhysGuestCR3 = pCtx->cr3;
3878 }
3879 else
3880 {
3881 /*
3882 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3883 * thinks it accesses physical memory directly, we use our identity-mapped
3884 * page table to map guest-linear to guest-physical addresses. EPT takes care
3885 * of translating it to host-physical addresses.
3886 */
3887 RTGCPHYS GCPhys;
3888 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3889
3890 /* We obtain it here every time as the guest could have relocated this PCI region. */
3891 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3892 if (RT_SUCCESS(rc))
3893 { /* likely */ }
3894 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3895 {
3896 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3897 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3898 }
3899 else
3900 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3901
3902 GCPhysGuestCR3 = GCPhys;
3903 }
3904
3905 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3906 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3907 AssertRCReturn(rc, rc);
3908 }
3909 else
3910 {
3911 /* Non-nested paging case, just use the hypervisor's CR3. */
3912 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3913
3914 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3915 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3916 AssertRCReturn(rc, rc);
3917 }
3918
3919 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3920 }
3921
3922 /*
3923 * Guest CR4.
3924 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3925 */
3926 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3927 {
3928 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3929 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3930 Assert(!RT_HI_U32(pCtx->cr4));
3931
3932 uint32_t u32GuestCr4 = pCtx->cr4;
3933 uint32_t const u32ShadowCr4 = pCtx->cr4;
3934
3935 /*
3936 * Setup VT-x's view of the guest CR4.
3937 *
3938 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3939 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3940 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3941 *
3942 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3943 */
3944 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3945 {
3946 Assert(pVM->hm.s.vmx.pRealModeTSS);
3947 Assert(PDMVmmDevHeapIsEnabled(pVM));
3948 u32GuestCr4 &= ~X86_CR4_VME;
3949 }
3950
3951 if (pVM->hm.s.fNestedPaging)
3952 {
3953 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3954 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3955 {
3956 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3957 u32GuestCr4 |= X86_CR4_PSE;
3958 /* Our identity mapping is a 32-bit page directory. */
3959 u32GuestCr4 &= ~X86_CR4_PAE;
3960 }
3961 /* else use guest CR4.*/
3962 }
3963 else
3964 {
3965 /*
3966 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3967 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3968 */
3969 switch (pVCpu->hm.s.enmShadowMode)
3970 {
3971 case PGMMODE_REAL: /* Real-mode. */
3972 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3973 case PGMMODE_32_BIT: /* 32-bit paging. */
3974 {
3975 u32GuestCr4 &= ~X86_CR4_PAE;
3976 break;
3977 }
3978
3979 case PGMMODE_PAE: /* PAE paging. */
3980 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3981 {
3982 u32GuestCr4 |= X86_CR4_PAE;
3983 break;
3984 }
3985
3986 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3987 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3988#ifdef VBOX_ENABLE_64_BITS_GUESTS
3989 break;
3990#endif
3991 default:
3992 AssertFailed();
3993 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3994 }
3995 }
3996
3997 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3998 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3999 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4000 u32GuestCr4 |= fSetCr4;
4001 u32GuestCr4 &= fZapCr4;
4002
4003 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4004 that would cause a VM-exit. */
4005 uint32_t u32Cr4Mask = X86_CR4_VME
4006 | X86_CR4_PAE
4007 | X86_CR4_PGE
4008 | X86_CR4_PSE
4009 | X86_CR4_VMXE;
4010 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4011 u32Cr4Mask |= X86_CR4_OSXSAVE;
4012 if (pVM->cpum.ro.GuestFeatures.fPcid)
4013 u32Cr4Mask |= X86_CR4_PCIDE;
4014
4015 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4016 into the VMCS and update our cache. */
4017 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4018 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4019 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4020 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4021 AssertRCReturn(rc, rc);
4022 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4023
4024 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4025 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4026
4027 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4028
4029 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4030 fZapCr4));
4031 }
4032 return rc;
4033}
4034
4035
4036/**
4037 * Exports the guest debug registers into the guest-state area in the VMCS.
4038 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4039 *
4040 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4041 *
4042 * @returns VBox status code.
4043 * @param pVCpu The cross context virtual CPU structure.
4044 *
4045 * @remarks No-long-jump zone!!!
4046 */
4047static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4048{
4049 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4050
4051#ifdef VBOX_STRICT
4052 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4053 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4054 {
4055 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4056 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4057 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4058 }
4059#endif
4060
4061 bool fSteppingDB = false;
4062 bool fInterceptMovDRx = false;
4063 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4064 if (pVCpu->hm.s.fSingleInstruction)
4065 {
4066 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4067 PVM pVM = pVCpu->CTX_SUFF(pVM);
4068 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4069 {
4070 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4071 Assert(fSteppingDB == false);
4072 }
4073 else
4074 {
4075 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4076 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4077 pVCpu->hm.s.fClearTrapFlag = true;
4078 fSteppingDB = true;
4079 }
4080 }
4081
4082 uint32_t u32GuestDr7;
4083 if ( fSteppingDB
4084 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4085 {
4086 /*
4087 * Use the combined guest and host DRx values found in the hypervisor register set
4088 * because the debugger has breakpoints active or someone is single stepping on the
4089 * host side without a monitor trap flag.
4090 *
4091 * Note! DBGF expects a clean DR6 state before executing guest code.
4092 */
4093#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4094 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4095 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4096 {
4097 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4098 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4099 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4100 }
4101 else
4102#endif
4103 if (!CPUMIsHyperDebugStateActive(pVCpu))
4104 {
4105 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4106 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4107 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4108 }
4109
4110 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4111 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4112 pVCpu->hm.s.fUsingHyperDR7 = true;
4113 fInterceptMovDRx = true;
4114 }
4115 else
4116 {
4117 /*
4118 * If the guest has enabled debug registers, we need to load them prior to
4119 * executing guest code so they'll trigger at the right time.
4120 */
4121 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4122 {
4123#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4124 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4125 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4126 {
4127 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4128 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4129 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4130 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4131 }
4132 else
4133#endif
4134 if (!CPUMIsGuestDebugStateActive(pVCpu))
4135 {
4136 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4137 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4138 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4139 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4140 }
4141 Assert(!fInterceptMovDRx);
4142 }
4143 /*
4144 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4145 * must intercept #DB in order to maintain a correct DR6 guest value, and
4146 * because we need to intercept it to prevent nested #DBs from hanging the
4147 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4148 */
4149#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4150 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4151 && !CPUMIsGuestDebugStateActive(pVCpu))
4152#else
4153 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4154#endif
4155 {
4156 fInterceptMovDRx = true;
4157 }
4158
4159 /* Update DR7 with the actual guest value. */
4160 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4161 pVCpu->hm.s.fUsingHyperDR7 = false;
4162 }
4163
4164 if (fInterceptMovDRx)
4165 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4166 else
4167 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4168
4169 /*
4170 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4171 * monitor-trap flag and update our cache.
4172 */
4173 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4174 {
4175 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4176 AssertRCReturn(rc2, rc2);
4177 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4178 }
4179
4180 /*
4181 * Update guest DR7.
4182 */
4183 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4184 AssertRCReturn(rc, rc);
4185
4186 return VINF_SUCCESS;
4187}
4188
4189
4190#ifdef VBOX_STRICT
4191/**
4192 * Strict function to validate segment registers.
4193 *
4194 * @param pVCpu The cross context virtual CPU structure.
4195 *
4196 * @remarks Will import guest CR0 on strict builds during validation of
4197 * segments.
4198 */
4199static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4200{
4201 /*
4202 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4203 *
4204 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4205 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4206 * and doesn't change the guest-context value.
4207 */
4208 PVM pVM = pVCpu->CTX_SUFF(pVM);
4209 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4210 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4211 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4212 && ( !CPUMIsGuestInRealModeEx(pCtx)
4213 && !CPUMIsGuestInV86ModeEx(pCtx)))
4214 {
4215 /* Protected mode checks */
4216 /* CS */
4217 Assert(pCtx->cs.Attr.n.u1Present);
4218 Assert(!(pCtx->cs.Attr.u & 0xf00));
4219 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4220 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4221 || !(pCtx->cs.Attr.n.u1Granularity));
4222 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4223 || (pCtx->cs.Attr.n.u1Granularity));
4224 /* CS cannot be loaded with NULL in protected mode. */
4225 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4226 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4227 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4228 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4229 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4230 else
4231 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4232 /* SS */
4233 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4234 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4235 if ( !(pCtx->cr0 & X86_CR0_PE)
4236 || pCtx->cs.Attr.n.u4Type == 3)
4237 {
4238 Assert(!pCtx->ss.Attr.n.u2Dpl);
4239 }
4240 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4241 {
4242 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4243 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4244 Assert(pCtx->ss.Attr.n.u1Present);
4245 Assert(!(pCtx->ss.Attr.u & 0xf00));
4246 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4247 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4248 || !(pCtx->ss.Attr.n.u1Granularity));
4249 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4250 || (pCtx->ss.Attr.n.u1Granularity));
4251 }
4252 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4253 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4254 {
4255 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4256 Assert(pCtx->ds.Attr.n.u1Present);
4257 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4258 Assert(!(pCtx->ds.Attr.u & 0xf00));
4259 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4260 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4261 || !(pCtx->ds.Attr.n.u1Granularity));
4262 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4263 || (pCtx->ds.Attr.n.u1Granularity));
4264 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4265 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4266 }
4267 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4268 {
4269 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4270 Assert(pCtx->es.Attr.n.u1Present);
4271 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4272 Assert(!(pCtx->es.Attr.u & 0xf00));
4273 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4274 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4275 || !(pCtx->es.Attr.n.u1Granularity));
4276 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4277 || (pCtx->es.Attr.n.u1Granularity));
4278 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4279 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4280 }
4281 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4282 {
4283 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4284 Assert(pCtx->fs.Attr.n.u1Present);
4285 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4286 Assert(!(pCtx->fs.Attr.u & 0xf00));
4287 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4288 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4289 || !(pCtx->fs.Attr.n.u1Granularity));
4290 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4291 || (pCtx->fs.Attr.n.u1Granularity));
4292 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4293 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4294 }
4295 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4296 {
4297 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4298 Assert(pCtx->gs.Attr.n.u1Present);
4299 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4300 Assert(!(pCtx->gs.Attr.u & 0xf00));
4301 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4302 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4303 || !(pCtx->gs.Attr.n.u1Granularity));
4304 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4305 || (pCtx->gs.Attr.n.u1Granularity));
4306 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4307 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4308 }
4309 /* 64-bit capable CPUs. */
4310# if HC_ARCH_BITS == 64
4311 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4312 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4313 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4314 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4315# endif
4316 }
4317 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4318 || ( CPUMIsGuestInRealModeEx(pCtx)
4319 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4320 {
4321 /* Real and v86 mode checks. */
4322 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4323 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4324 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4325 {
4326 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4327 }
4328 else
4329 {
4330 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4331 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4332 }
4333
4334 /* CS */
4335 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4336 Assert(pCtx->cs.u32Limit == 0xffff);
4337 Assert(u32CSAttr == 0xf3);
4338 /* SS */
4339 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4340 Assert(pCtx->ss.u32Limit == 0xffff);
4341 Assert(u32SSAttr == 0xf3);
4342 /* DS */
4343 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4344 Assert(pCtx->ds.u32Limit == 0xffff);
4345 Assert(u32DSAttr == 0xf3);
4346 /* ES */
4347 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4348 Assert(pCtx->es.u32Limit == 0xffff);
4349 Assert(u32ESAttr == 0xf3);
4350 /* FS */
4351 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4352 Assert(pCtx->fs.u32Limit == 0xffff);
4353 Assert(u32FSAttr == 0xf3);
4354 /* GS */
4355 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4356 Assert(pCtx->gs.u32Limit == 0xffff);
4357 Assert(u32GSAttr == 0xf3);
4358 /* 64-bit capable CPUs. */
4359# if HC_ARCH_BITS == 64
4360 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4361 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4362 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4363 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4364# endif
4365 }
4366}
4367#endif /* VBOX_STRICT */
4368
4369
4370/**
4371 * Exports a guest segment register into the guest-state area in the VMCS.
4372 *
4373 * @returns VBox status code.
4374 * @param pVCpu The cross context virtual CPU structure.
4375 * @param idxSel Index of the selector in the VMCS.
4376 * @param idxLimit Index of the segment limit in the VMCS.
4377 * @param idxBase Index of the segment base in the VMCS.
4378 * @param idxAccess Index of the access rights of the segment in the VMCS.
4379 * @param pSelReg Pointer to the segment selector.
4380 *
4381 * @remarks No-long-jump zone!!!
4382 */
4383static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4384 PCCPUMSELREG pSelReg)
4385{
4386 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4387 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4388 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4389 AssertRCReturn(rc, rc);
4390
4391 uint32_t u32Access = pSelReg->Attr.u;
4392 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4393 {
4394 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4395 u32Access = 0xf3;
4396 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4397 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4398 }
4399 else
4400 {
4401 /*
4402 * The way to differentiate between whether this is really a null selector or was just
4403 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4404 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4405 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4406 * NULL selectors loaded in protected-mode have their attribute as 0.
4407 */
4408 if (!u32Access)
4409 u32Access = X86DESCATTR_UNUSABLE;
4410 }
4411
4412 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4413 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4414 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4415
4416 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4417 AssertRCReturn(rc, rc);
4418 return rc;
4419}
4420
4421
4422/**
4423 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4424 * into the guest-state area in the VMCS.
4425 *
4426 * @returns VBox status code.
4427 * @param pVCpu The cross context virtual CPU structure.
4428 *
4429 * @remarks Will import guest CR0 on strict builds during validation of
4430 * segments.
4431 * @remarks No-long-jump zone!!!
4432 */
4433static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4434{
4435 int rc = VERR_INTERNAL_ERROR_5;
4436 PVM pVM = pVCpu->CTX_SUFF(pVM);
4437 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4438
4439 /*
4440 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4441 */
4442 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4443 {
4444#ifdef VBOX_WITH_REM
4445 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4446 {
4447 Assert(pVM->hm.s.vmx.pRealModeTSS);
4448 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4449 if ( pVCpu->hm.s.vmx.fWasInRealMode
4450 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4451 {
4452 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4453 in real-mode (e.g. OpenBSD 4.0) */
4454 REMFlushTBs(pVM);
4455 Log4Func(("Switch to protected mode detected!\n"));
4456 pVCpu->hm.s.vmx.fWasInRealMode = false;
4457 }
4458 }
4459#endif
4460 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4461 {
4462 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4463 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4464 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4465 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4466 AssertRCReturn(rc, rc);
4467 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4468 }
4469
4470 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4471 {
4472 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4473 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4474 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4475 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4476 AssertRCReturn(rc, rc);
4477 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4478 }
4479
4480 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4481 {
4482 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4483 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4484 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4485 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4486 AssertRCReturn(rc, rc);
4487 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4488 }
4489
4490 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4491 {
4492 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4493 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4494 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4495 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4496 AssertRCReturn(rc, rc);
4497 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4498 }
4499
4500 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4501 {
4502 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4503 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4504 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4505 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4506 AssertRCReturn(rc, rc);
4507 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4508 }
4509
4510 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4511 {
4512 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4513 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4514 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4515 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4516 AssertRCReturn(rc, rc);
4517 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4518 }
4519
4520#ifdef VBOX_STRICT
4521 hmR0VmxValidateSegmentRegs(pVCpu);
4522#endif
4523
4524 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4525 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4526 }
4527
4528 /*
4529 * Guest TR.
4530 */
4531 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4532 {
4533 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4534
4535 /*
4536 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4537 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4538 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4539 */
4540 uint16_t u16Sel = 0;
4541 uint32_t u32Limit = 0;
4542 uint64_t u64Base = 0;
4543 uint32_t u32AccessRights = 0;
4544
4545 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4546 {
4547 u16Sel = pCtx->tr.Sel;
4548 u32Limit = pCtx->tr.u32Limit;
4549 u64Base = pCtx->tr.u64Base;
4550 u32AccessRights = pCtx->tr.Attr.u;
4551 }
4552 else
4553 {
4554 Assert(pVM->hm.s.vmx.pRealModeTSS);
4555 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4556
4557 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4558 RTGCPHYS GCPhys;
4559 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4560 AssertRCReturn(rc, rc);
4561
4562 X86DESCATTR DescAttr;
4563 DescAttr.u = 0;
4564 DescAttr.n.u1Present = 1;
4565 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4566
4567 u16Sel = 0;
4568 u32Limit = HM_VTX_TSS_SIZE;
4569 u64Base = GCPhys; /* in real-mode phys = virt. */
4570 u32AccessRights = DescAttr.u;
4571 }
4572
4573 /* Validate. */
4574 Assert(!(u16Sel & RT_BIT(2)));
4575 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4576 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4577 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4578 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4579 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4580 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4581 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4582 Assert( (u32Limit & 0xfff) == 0xfff
4583 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4584 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4585 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4586
4587 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4588 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4589 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4590 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4591 AssertRCReturn(rc, rc);
4592
4593 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4594 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4595 }
4596
4597 /*
4598 * Guest GDTR.
4599 */
4600 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4601 {
4602 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4603
4604 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4605 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4606 AssertRCReturn(rc, rc);
4607
4608 /* Validate. */
4609 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4610
4611 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4612 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4613 }
4614
4615 /*
4616 * Guest LDTR.
4617 */
4618 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4619 {
4620 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4621
4622 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4623 uint32_t u32Access = 0;
4624 if (!pCtx->ldtr.Attr.u)
4625 u32Access = X86DESCATTR_UNUSABLE;
4626 else
4627 u32Access = pCtx->ldtr.Attr.u;
4628
4629 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4630 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4631 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4632 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4633 AssertRCReturn(rc, rc);
4634
4635 /* Validate. */
4636 if (!(u32Access & X86DESCATTR_UNUSABLE))
4637 {
4638 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4639 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4640 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4641 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4642 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4643 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4644 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4645 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4646 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4647 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4648 }
4649
4650 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4651 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4652 }
4653
4654 /*
4655 * Guest IDTR.
4656 */
4657 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4658 {
4659 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4660
4661 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4662 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4663 AssertRCReturn(rc, rc);
4664
4665 /* Validate. */
4666 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4667
4668 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4669 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4670 }
4671
4672 return VINF_SUCCESS;
4673}
4674
4675
4676/**
4677 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4678 * areas.
4679 *
4680 * These MSRs will automatically be loaded to the host CPU on every successful
4681 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4682 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4683 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4684 *
4685 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4686 *
4687 * @returns VBox status code.
4688 * @param pVCpu The cross context virtual CPU structure.
4689 *
4690 * @remarks No-long-jump zone!!!
4691 */
4692static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4693{
4694 AssertPtr(pVCpu);
4695 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4696
4697 /*
4698 * MSRs that we use the auto-load/store MSR area in the VMCS.
4699 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4700 */
4701 PVM pVM = pVCpu->CTX_SUFF(pVM);
4702 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4703 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4704 {
4705 if (pVM->hm.s.fAllow64BitGuests)
4706 {
4707#if HC_ARCH_BITS == 32
4708 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4709
4710 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4711 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4712 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4713 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4714 AssertRCReturn(rc, rc);
4715# ifdef LOG_ENABLED
4716 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4717 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4718 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4719# endif
4720#endif
4721 }
4722 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4723 }
4724
4725 /*
4726 * Guest Sysenter MSRs.
4727 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4728 * VM-exits on WRMSRs for these MSRs.
4729 */
4730 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4731 {
4732 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4733
4734 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4735 {
4736 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4737 AssertRCReturn(rc, rc);
4738 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4739 }
4740
4741 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4742 {
4743 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4744 AssertRCReturn(rc, rc);
4745 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4746 }
4747
4748 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4749 {
4750 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4751 AssertRCReturn(rc, rc);
4752 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4753 }
4754 }
4755
4756 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4757 {
4758 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4759
4760 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4761 {
4762 /*
4763 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4764 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4765 */
4766 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4767 {
4768 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4769 AssertRCReturn(rc,rc);
4770 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4771 }
4772 else
4773 {
4774 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4775 NULL /* pfAddedAndUpdated */);
4776 AssertRCReturn(rc, rc);
4777
4778 /* We need to intercept reads too, see @bugref{7386#c16}. */
4779 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4780 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4781 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4782 pVCpu->hm.s.vmx.cMsrs));
4783 }
4784 }
4785 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4786 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4787 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4788 }
4789
4790 return VINF_SUCCESS;
4791}
4792
4793
4794#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4795/**
4796 * Check if guest state allows safe use of 32-bit switcher again.
4797 *
4798 * Segment bases and protected mode structures must be 32-bit addressable
4799 * because the 32-bit switcher will ignore high dword when writing these VMCS
4800 * fields. See @bugref{8432} for details.
4801 *
4802 * @returns true if safe, false if must continue to use the 64-bit switcher.
4803 * @param pCtx Pointer to the guest-CPU context.
4804 *
4805 * @remarks No-long-jump zone!!!
4806 */
4807static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4808{
4809 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4810 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4811 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4812 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4813 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4814 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4815 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4816 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4817 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4818 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4819
4820 /* All good, bases are 32-bit. */
4821 return true;
4822}
4823#endif
4824
4825
4826/**
4827 * Selects up the appropriate function to run guest code.
4828 *
4829 * @returns VBox status code.
4830 * @param pVCpu The cross context virtual CPU structure.
4831 *
4832 * @remarks No-long-jump zone!!!
4833 */
4834static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4835{
4836 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4837 if (CPUMIsGuestInLongModeEx(pCtx))
4838 {
4839#ifndef VBOX_ENABLE_64_BITS_GUESTS
4840 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4841#endif
4842 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4843#if HC_ARCH_BITS == 32
4844 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4845 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4846 {
4847#ifdef VBOX_STRICT
4848 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4849 {
4850 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4851 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4852 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4853 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4854 | HM_CHANGED_VMX_ENTRY_CTLS
4855 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4856 }
4857#endif
4858 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4859
4860 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4861 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4862 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4863 Log4Func(("Selected 64-bit switcher\n"));
4864 }
4865#else
4866 /* 64-bit host. */
4867 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4868#endif
4869 }
4870 else
4871 {
4872 /* Guest is not in long mode, use the 32-bit handler. */
4873#if HC_ARCH_BITS == 32
4874 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4875 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4876 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4877 {
4878# ifdef VBOX_STRICT
4879 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4880 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4881 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4882 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4883 | HM_CHANGED_VMX_ENTRY_CTLS
4884 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4885# endif
4886 }
4887# ifdef VBOX_ENABLE_64_BITS_GUESTS
4888 /*
4889 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4890 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4891 * switcher flag because now we know the guest is in a sane state where it's safe
4892 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4893 * the much faster 32-bit switcher again.
4894 */
4895 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4896 {
4897 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4898 Log4Func(("Selected 32-bit switcher\n"));
4899 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4900 }
4901 else
4902 {
4903 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4904 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4905 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4906 {
4907 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4908 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4909 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4910 | HM_CHANGED_VMX_ENTRY_CTLS
4911 | HM_CHANGED_VMX_EXIT_CTLS
4912 | HM_CHANGED_HOST_CONTEXT);
4913 Log4Func(("Selected 32-bit switcher (safe)\n"));
4914 }
4915 }
4916# else
4917 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4918# endif
4919#else
4920 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4921#endif
4922 }
4923 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4924 return VINF_SUCCESS;
4925}
4926
4927
4928/**
4929 * Wrapper for running the guest code in VT-x.
4930 *
4931 * @returns VBox status code, no informational status codes.
4932 * @param pVCpu The cross context virtual CPU structure.
4933 *
4934 * @remarks No-long-jump zone!!!
4935 */
4936DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4937{
4938 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4939 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4940 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4941
4942 /*
4943 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4944 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4945 * callee-saved and thus the need for this XMM wrapper.
4946 *
4947 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4948 */
4949 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4950 /** @todo Add stats for resume vs launch. */
4951 PVM pVM = pVCpu->CTX_SUFF(pVM);
4952#ifdef VBOX_WITH_KERNEL_USING_XMM
4953 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4954#else
4955 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4956#endif
4957 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4958 return rc;
4959}
4960
4961
4962/**
4963 * Reports world-switch error and dumps some useful debug info.
4964 *
4965 * @param pVCpu The cross context virtual CPU structure.
4966 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4967 * @param pVmxTransient Pointer to the VMX transient structure (only
4968 * exitReason updated).
4969 */
4970static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4971{
4972 Assert(pVCpu);
4973 Assert(pVmxTransient);
4974 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4975
4976 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4977 switch (rcVMRun)
4978 {
4979 case VERR_VMX_INVALID_VMXON_PTR:
4980 AssertFailed();
4981 break;
4982 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4983 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4984 {
4985 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4986 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4987 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
4988 AssertRC(rc);
4989
4990 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4991 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4992 Cannot do it here as we may have been long preempted. */
4993
4994#ifdef VBOX_STRICT
4995 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4996 pVmxTransient->uExitReason));
4997 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
4998 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4999 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5000 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5001 else
5002 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5003 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5004 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5005
5006 /* VMX control bits. */
5007 uint32_t u32Val;
5008 uint64_t u64Val;
5009 RTHCUINTREG uHCReg;
5010 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5011 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5012 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5013 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5014 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5015 {
5016 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5017 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5018 }
5019 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5020 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5021 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5022 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5023 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5024 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5025 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5026 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5027 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5028 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5029 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5030 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5031 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5032 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5033 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5034 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5035 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5036 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5037 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5038 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5039 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5040 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5041 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5042 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5043 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5044 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5045 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5046 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5047 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5048 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5049 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5050 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5051 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5052 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5053 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5054 {
5055 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5056 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5057 }
5058
5059 /* Guest bits. */
5060 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5061 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5062 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5063 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5064 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5065 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5066 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5067 {
5068 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5069 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5070 }
5071
5072 /* Host bits. */
5073 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5074 Log4(("Host CR0 %#RHr\n", uHCReg));
5075 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5076 Log4(("Host CR3 %#RHr\n", uHCReg));
5077 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5078 Log4(("Host CR4 %#RHr\n", uHCReg));
5079
5080 RTGDTR HostGdtr;
5081 PCX86DESCHC pDesc;
5082 ASMGetGDTR(&HostGdtr);
5083 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5084 Log4(("Host CS %#08x\n", u32Val));
5085 if (u32Val < HostGdtr.cbGdt)
5086 {
5087 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5088 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5089 }
5090
5091 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5092 Log4(("Host DS %#08x\n", u32Val));
5093 if (u32Val < HostGdtr.cbGdt)
5094 {
5095 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5096 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5097 }
5098
5099 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5100 Log4(("Host ES %#08x\n", u32Val));
5101 if (u32Val < HostGdtr.cbGdt)
5102 {
5103 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5104 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5105 }
5106
5107 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5108 Log4(("Host FS %#08x\n", u32Val));
5109 if (u32Val < HostGdtr.cbGdt)
5110 {
5111 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5112 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5113 }
5114
5115 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5116 Log4(("Host GS %#08x\n", u32Val));
5117 if (u32Val < HostGdtr.cbGdt)
5118 {
5119 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5120 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5121 }
5122
5123 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5124 Log4(("Host SS %#08x\n", u32Val));
5125 if (u32Val < HostGdtr.cbGdt)
5126 {
5127 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5128 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5129 }
5130
5131 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5132 Log4(("Host TR %#08x\n", u32Val));
5133 if (u32Val < HostGdtr.cbGdt)
5134 {
5135 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5136 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5137 }
5138
5139 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5140 Log4(("Host TR Base %#RHv\n", uHCReg));
5141 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5142 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5143 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5144 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5145 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5146 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5147 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5148 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5149 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5150 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5151 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5152 Log4(("Host RSP %#RHv\n", uHCReg));
5153 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5154 Log4(("Host RIP %#RHv\n", uHCReg));
5155# if HC_ARCH_BITS == 64
5156 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5157 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5158 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5159 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5160 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5161 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5162# endif
5163#endif /* VBOX_STRICT */
5164 break;
5165 }
5166
5167 default:
5168 /* Impossible */
5169 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5170 break;
5171 }
5172}
5173
5174
5175#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5176#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5177# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5178#endif
5179#ifdef VBOX_STRICT
5180static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5181{
5182 switch (idxField)
5183 {
5184 case VMX_VMCS_GUEST_RIP:
5185 case VMX_VMCS_GUEST_RSP:
5186 case VMX_VMCS_GUEST_SYSENTER_EIP:
5187 case VMX_VMCS_GUEST_SYSENTER_ESP:
5188 case VMX_VMCS_GUEST_GDTR_BASE:
5189 case VMX_VMCS_GUEST_IDTR_BASE:
5190 case VMX_VMCS_GUEST_CS_BASE:
5191 case VMX_VMCS_GUEST_DS_BASE:
5192 case VMX_VMCS_GUEST_ES_BASE:
5193 case VMX_VMCS_GUEST_FS_BASE:
5194 case VMX_VMCS_GUEST_GS_BASE:
5195 case VMX_VMCS_GUEST_SS_BASE:
5196 case VMX_VMCS_GUEST_LDTR_BASE:
5197 case VMX_VMCS_GUEST_TR_BASE:
5198 case VMX_VMCS_GUEST_CR3:
5199 return true;
5200 }
5201 return false;
5202}
5203
5204static bool hmR0VmxIsValidReadField(uint32_t idxField)
5205{
5206 switch (idxField)
5207 {
5208 /* Read-only fields. */
5209 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5210 return true;
5211 }
5212 /* Remaining readable fields should also be writable. */
5213 return hmR0VmxIsValidWriteField(idxField);
5214}
5215#endif /* VBOX_STRICT */
5216
5217
5218/**
5219 * Executes the specified handler in 64-bit mode.
5220 *
5221 * @returns VBox status code (no informational status codes).
5222 * @param pVCpu The cross context virtual CPU structure.
5223 * @param enmOp The operation to perform.
5224 * @param cParams Number of parameters.
5225 * @param paParam Array of 32-bit parameters.
5226 */
5227VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5228{
5229 PVM pVM = pVCpu->CTX_SUFF(pVM);
5230 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5231 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5232 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5233 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5234
5235#ifdef VBOX_STRICT
5236 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5237 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5238
5239 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5240 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5241#endif
5242
5243 /* Disable interrupts. */
5244 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5245
5246#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5247 RTCPUID idHostCpu = RTMpCpuId();
5248 CPUMR0SetLApic(pVCpu, idHostCpu);
5249#endif
5250
5251 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5252 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5253
5254 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5255 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5256 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5257
5258 /* Leave VMX Root Mode. */
5259 VMXDisable();
5260
5261 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5262
5263 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5264 CPUMSetHyperEIP(pVCpu, enmOp);
5265 for (int i = (int)cParams - 1; i >= 0; i--)
5266 CPUMPushHyper(pVCpu, paParam[i]);
5267
5268 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5269
5270 /* Call the switcher. */
5271 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5272 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5273
5274 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5275 /* Make sure the VMX instructions don't cause #UD faults. */
5276 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5277
5278 /* Re-enter VMX Root Mode */
5279 int rc2 = VMXEnable(HCPhysCpuPage);
5280 if (RT_FAILURE(rc2))
5281 {
5282 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5283 ASMSetFlags(fOldEFlags);
5284 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5285 return rc2;
5286 }
5287
5288 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5289 AssertRC(rc2);
5290 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5291 Assert(!(ASMGetFlags() & X86_EFL_IF));
5292 ASMSetFlags(fOldEFlags);
5293 return rc;
5294}
5295
5296
5297/**
5298 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5299 * supporting 64-bit guests.
5300 *
5301 * @returns VBox status code.
5302 * @param fResume Whether to VMLAUNCH or VMRESUME.
5303 * @param pCtx Pointer to the guest-CPU context.
5304 * @param pCache Pointer to the VMCS cache.
5305 * @param pVM The cross context VM structure.
5306 * @param pVCpu The cross context virtual CPU structure.
5307 */
5308DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5309{
5310 NOREF(fResume);
5311
5312 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5313 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5314
5315#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5316 pCache->uPos = 1;
5317 pCache->interPD = PGMGetInterPaeCR3(pVM);
5318 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5319#endif
5320
5321#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5322 pCache->TestIn.HCPhysCpuPage = 0;
5323 pCache->TestIn.HCPhysVmcs = 0;
5324 pCache->TestIn.pCache = 0;
5325 pCache->TestOut.HCPhysVmcs = 0;
5326 pCache->TestOut.pCache = 0;
5327 pCache->TestOut.pCtx = 0;
5328 pCache->TestOut.eflags = 0;
5329#else
5330 NOREF(pCache);
5331#endif
5332
5333 uint32_t aParam[10];
5334 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5335 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5336 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5337 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5338 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5339 aParam[5] = 0;
5340 aParam[6] = VM_RC_ADDR(pVM, pVM);
5341 aParam[7] = 0;
5342 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5343 aParam[9] = 0;
5344
5345#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5346 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5347 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5348#endif
5349 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5350
5351#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5352 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5353 Assert(pCtx->dr[4] == 10);
5354 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5355#endif
5356
5357#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5358 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5359 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5360 pVCpu->hm.s.vmx.HCPhysVmcs));
5361 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5362 pCache->TestOut.HCPhysVmcs));
5363 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5364 pCache->TestOut.pCache));
5365 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5366 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5367 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5368 pCache->TestOut.pCtx));
5369 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5370#endif
5371 NOREF(pCtx);
5372 return rc;
5373}
5374
5375
5376/**
5377 * Initialize the VMCS-Read cache.
5378 *
5379 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5380 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5381 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5382 * (those that have a 32-bit FULL & HIGH part).
5383 *
5384 * @returns VBox status code.
5385 * @param pVCpu The cross context virtual CPU structure.
5386 */
5387static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5388{
5389#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5390 do { \
5391 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5392 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5393 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5394 ++cReadFields; \
5395 } while (0)
5396
5397 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5398 uint32_t cReadFields = 0;
5399
5400 /*
5401 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5402 * and serve to indicate exceptions to the rules.
5403 */
5404
5405 /* Guest-natural selector base fields. */
5406#if 0
5407 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5408 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5409 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5410#endif
5411 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5412 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5413 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5414 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5416 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5417 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5418 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5419 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5420 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5421 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5422 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5423#if 0
5424 /* Unused natural width guest-state fields. */
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5427#endif
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5430
5431 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5432 these 64-bit fields (using "FULL" and "HIGH" fields). */
5433#if 0
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5438 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5442 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5443#endif
5444
5445 /* Natural width guest-state fields. */
5446 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5447#if 0
5448 /* Currently unused field. */
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5450#endif
5451
5452 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5453 {
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5455 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5456 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5457 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5458 }
5459 else
5460 {
5461 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5462 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5463 }
5464
5465#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5466 return VINF_SUCCESS;
5467}
5468
5469
5470/**
5471 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5472 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5473 * darwin, running 64-bit guests).
5474 *
5475 * @returns VBox status code.
5476 * @param pVCpu The cross context virtual CPU structure.
5477 * @param idxField The VMCS field encoding.
5478 * @param u64Val 16, 32 or 64-bit value.
5479 */
5480VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5481{
5482 int rc;
5483 switch (idxField)
5484 {
5485 /*
5486 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5487 */
5488 /* 64-bit Control fields. */
5489 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5490 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5491 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5492 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5493 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5494 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5495 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5496 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5497 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5498 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5499 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5500 case VMX_VMCS64_CTRL_EPTP_FULL:
5501 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5502 /* 64-bit Guest-state fields. */
5503 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5504 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5505 case VMX_VMCS64_GUEST_PAT_FULL:
5506 case VMX_VMCS64_GUEST_EFER_FULL:
5507 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5508 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5509 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5510 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5511 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5512 /* 64-bit Host-state fields. */
5513 case VMX_VMCS64_HOST_PAT_FULL:
5514 case VMX_VMCS64_HOST_EFER_FULL:
5515 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5516 {
5517 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5518 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5519 break;
5520 }
5521
5522 /*
5523 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5524 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5525 */
5526 /* Natural-width Guest-state fields. */
5527 case VMX_VMCS_GUEST_CR3:
5528 case VMX_VMCS_GUEST_ES_BASE:
5529 case VMX_VMCS_GUEST_CS_BASE:
5530 case VMX_VMCS_GUEST_SS_BASE:
5531 case VMX_VMCS_GUEST_DS_BASE:
5532 case VMX_VMCS_GUEST_FS_BASE:
5533 case VMX_VMCS_GUEST_GS_BASE:
5534 case VMX_VMCS_GUEST_LDTR_BASE:
5535 case VMX_VMCS_GUEST_TR_BASE:
5536 case VMX_VMCS_GUEST_GDTR_BASE:
5537 case VMX_VMCS_GUEST_IDTR_BASE:
5538 case VMX_VMCS_GUEST_RSP:
5539 case VMX_VMCS_GUEST_RIP:
5540 case VMX_VMCS_GUEST_SYSENTER_ESP:
5541 case VMX_VMCS_GUEST_SYSENTER_EIP:
5542 {
5543 if (!(RT_HI_U32(u64Val)))
5544 {
5545 /* If this field is 64-bit, VT-x will zero out the top bits. */
5546 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5547 }
5548 else
5549 {
5550 /* Assert that only the 32->64 switcher case should ever come here. */
5551 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5552 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5553 }
5554 break;
5555 }
5556
5557 default:
5558 {
5559 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5560 rc = VERR_INVALID_PARAMETER;
5561 break;
5562 }
5563 }
5564 AssertRCReturn(rc, rc);
5565 return rc;
5566}
5567
5568
5569/**
5570 * Queue up a VMWRITE by using the VMCS write cache.
5571 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5572 *
5573 * @param pVCpu The cross context virtual CPU structure.
5574 * @param idxField The VMCS field encoding.
5575 * @param u64Val 16, 32 or 64-bit value.
5576 */
5577VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5578{
5579 AssertPtr(pVCpu);
5580 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5581
5582 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5583 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5584
5585 /* Make sure there are no duplicates. */
5586 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5587 {
5588 if (pCache->Write.aField[i] == idxField)
5589 {
5590 pCache->Write.aFieldVal[i] = u64Val;
5591 return VINF_SUCCESS;
5592 }
5593 }
5594
5595 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5596 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5597 pCache->Write.cValidEntries++;
5598 return VINF_SUCCESS;
5599}
5600#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5601
5602
5603/**
5604 * Sets up the usage of TSC-offsetting and updates the VMCS.
5605 *
5606 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5607 * VMX preemption timer.
5608 *
5609 * @returns VBox status code.
5610 * @param pVCpu The cross context virtual CPU structure.
5611 *
5612 * @remarks No-long-jump zone!!!
5613 */
5614static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5615{
5616 bool fOffsettedTsc;
5617 bool fParavirtTsc;
5618 PVM pVM = pVCpu->CTX_SUFF(pVM);
5619 uint64_t uTscOffset;
5620 if (pVM->hm.s.vmx.fUsePreemptTimer)
5621 {
5622 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5623
5624 /* Make sure the returned values have sane upper and lower boundaries. */
5625 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5626 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5627 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5628 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5629
5630 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5631 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5632 AssertRC(rc);
5633 }
5634 else
5635 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5636
5637 if (fParavirtTsc)
5638 {
5639 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5640 information before every VM-entry, hence disable it for performance sake. */
5641#if 0
5642 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5643 AssertRC(rc);
5644#endif
5645 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5646 }
5647
5648 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5649 if ( fOffsettedTsc
5650 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5651 {
5652 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5653 {
5654 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5655 AssertRC(rc);
5656 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5657 }
5658
5659 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5660 {
5661 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5662 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5663 AssertRC(rc);
5664 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5665 }
5666 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5667 }
5668 else
5669 {
5670 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5671 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5672 {
5673 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5674 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5675 AssertRC(rc);
5676 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5677 }
5678 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5679 }
5680}
5681
5682
5683/**
5684 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5685 * VM-exit interruption info type.
5686 *
5687 * @returns The IEM exception flags.
5688 * @param uVector The event vector.
5689 * @param uVmxVectorType The VMX event type.
5690 *
5691 * @remarks This function currently only constructs flags required for
5692 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5693 * and CR2 aspects of an exception are not included).
5694 */
5695static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5696{
5697 uint32_t fIemXcptFlags;
5698 switch (uVmxVectorType)
5699 {
5700 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5701 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5702 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5703 break;
5704
5705 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5706 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5707 break;
5708
5709 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5710 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5711 break;
5712
5713 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5714 {
5715 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5716 if (uVector == X86_XCPT_BP)
5717 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5718 else if (uVector == X86_XCPT_OF)
5719 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5720 else
5721 {
5722 fIemXcptFlags = 0;
5723 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5724 }
5725 break;
5726 }
5727
5728 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5729 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5730 break;
5731
5732 default:
5733 fIemXcptFlags = 0;
5734 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5735 break;
5736 }
5737 return fIemXcptFlags;
5738}
5739
5740
5741/**
5742 * Sets an event as a pending event to be injected into the guest.
5743 *
5744 * @param pVCpu The cross context virtual CPU structure.
5745 * @param u32IntInfo The VM-entry interruption-information field.
5746 * @param cbInstr The VM-entry instruction length in bytes (for software
5747 * interrupts, exceptions and privileged software
5748 * exceptions).
5749 * @param u32ErrCode The VM-entry exception error code.
5750 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5751 * page-fault.
5752 *
5753 * @remarks Statistics counter assumes this is a guest event being injected or
5754 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5755 * always incremented.
5756 */
5757DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5758 RTGCUINTPTR GCPtrFaultAddress)
5759{
5760 Assert(!pVCpu->hm.s.Event.fPending);
5761 pVCpu->hm.s.Event.fPending = true;
5762 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5763 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5764 pVCpu->hm.s.Event.cbInstr = cbInstr;
5765 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5766}
5767
5768
5769/**
5770 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5771 *
5772 * @param pVCpu The cross context virtual CPU structure.
5773 */
5774DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5775{
5776 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5777 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5778 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5779 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5780 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5781}
5782
5783
5784/**
5785 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5786 *
5787 * @param pVCpu The cross context virtual CPU structure.
5788 */
5789DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5790{
5791 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5792 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5793 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5794 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5795 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5796}
5797
5798
5799/**
5800 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5801 *
5802 * @param pVCpu The cross context virtual CPU structure.
5803 */
5804DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5805{
5806 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5807 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5808 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5809 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5810 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5811}
5812
5813
5814/**
5815 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
5816 *
5817 * @param pVCpu The cross context virtual CPU structure.
5818 * @param cbInstr The value of RIP that is to be pushed on the guest
5819 * stack.
5820 */
5821DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, uint32_t cbInstr)
5822{
5823 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_OF)
5824 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_SW_INT)
5825 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5826 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5827 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5828}
5829
5830
5831/**
5832 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5833 *
5834 * @param pVCpu The cross context virtual CPU structure.
5835 * @param u32ErrCode The error code for the general-protection exception.
5836 */
5837DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5838{
5839 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5840 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5841 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5842 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5843 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5844}
5845
5846
5847/**
5848 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
5849 *
5850 * @param pVCpu The cross context virtual CPU structure.
5851 * @param u32ErrCode The error code for the stack exception.
5852 */
5853DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5854{
5855 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5856 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5857 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5858 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5859 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5860}
5861
5862
5863#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5864
5865/**
5866 * Decodes the memory operand of a VM-exit due to instruction execution.
5867 *
5868 * For instructions with two operands, the second operand is usually found in the
5869 * VM-exit qualification field.
5870 *
5871 * @returns Strict VBox status code (i.e. informational status codes too).
5872 * @retval VINF_SUCCESS if the operand was successfully decoded.
5873 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5874 * operand.
5875 * @param pVCpu The cross context virtual CPU structure.
5876 * @param pExitInstrInfo Pointer to the VM-exit instruction information.
5877 * @param fIsWrite Whether the operand is a destination memory operand
5878 * (i.e. writeable memory location) or not.
5879 * @param GCPtrDisp The instruction displacement field, if any. For
5880 * RIP-relative addressing pass RIP + displacement here.
5881 * @param pGCPtrMem Where to store the destination memory operand.
5882 */
5883static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, PCVMXEXITINSTRINFO pExitInstrInfo, RTGCPTR GCPtrDisp, bool fIsWrite,
5884 PRTGCPTR pGCPtrMem)
5885{
5886 Assert(pExitInstrInfo);
5887 Assert(pGCPtrMem);
5888 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5889
5890 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5891 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5892 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5893
5894 uint8_t const uAddrSize = pExitInstrInfo->InvVmxXsaves.u3AddrSize;
5895 uint8_t const iSegReg = pExitInstrInfo->InvVmxXsaves.iSegReg;
5896 bool const fIdxRegValid = !pExitInstrInfo->InvVmxXsaves.fIdxRegInvalid;
5897 uint8_t const iIdxReg = pExitInstrInfo->InvVmxXsaves.iIdxReg;
5898 uint8_t const uScale = pExitInstrInfo->InvVmxXsaves.u2Scaling;
5899 bool const fBaseRegValid = !pExitInstrInfo->InvVmxXsaves.fBaseRegInvalid;
5900 uint8_t const iBaseReg = pExitInstrInfo->InvVmxXsaves.iBaseReg;
5901 bool const fIsMemOperand = !pExitInstrInfo->InvVmxXsaves.fIsRegOperand;
5902 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5903
5904 /*
5905 * Validate instruction information.
5906 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5907 */
5908 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5909 ("Invalid address size. ExitInstrInfo=%#RX32\n", pExitInstrInfo->u), VERR_VMX_IPE_1);
5910 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5911 ("Invalid segment register. ExitInstrInfo=%#RX32\n", pExitInstrInfo->u), VERR_VMX_IPE_2);
5912 AssertLogRelMsgReturn(fIsMemOperand,
5913 ("Expected memory operand. ExitInstrInfo=%#RX32\n", pExitInstrInfo->u), VERR_VMX_IPE_3);
5914
5915 /*
5916 * Compute the complete effective address.
5917 *
5918 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
5919 * See AMD spec. 4.5.2 "Segment Registers".
5920 */
5921 RTGCPTR GCPtrMem = GCPtrDisp;
5922 if (fBaseRegValid)
5923 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5924 if (fIdxRegValid)
5925 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5926
5927 RTGCPTR const GCPtrOff = GCPtrMem;
5928 if ( !fIsLongMode
5929 || iSegReg >= X86_SREG_FS)
5930 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5931 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5932
5933 /*
5934 * Validate effective address.
5935 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5936 */
5937 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5938 Assert(cbAccess > 0);
5939 if (fIsLongMode)
5940 {
5941 if (X86_IS_CANONICAL(GCPtrMem))
5942 {
5943 *pGCPtrMem = GCPtrMem;
5944 return VINF_SUCCESS;
5945 }
5946
5947 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5948 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5949 return VINF_HM_PENDING_XCPT;
5950 }
5951
5952 /*
5953 * This is a watered down version of iemMemApplySegment().
5954 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
5955 * and segment CPL/DPL checks are skipped.
5956 */
5957 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5958 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5959 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
5960
5961 /* Check if the segment is present and usable. */
5962 if ( pSel->Attr.n.u1Present
5963 && !pSel->Attr.n.u1Unusable)
5964 {
5965 Assert(pSel->Attr.n.u1DescType);
5966 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5967 {
5968 /* Check permissions for the data segment. */
5969 if ( fIsWrite
5970 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
5971 {
5972 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
5973 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
5974 return VINF_HM_PENDING_XCPT;
5975 }
5976
5977 /* Check limits if it's a normal data segment. */
5978 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5979 {
5980 if ( GCPtrFirst32 > pSel->u32Limit
5981 || GCPtrLast32 > pSel->u32Limit)
5982 {
5983 Log4Func(("Data segment limit exceeded."
5984 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5985 GCPtrLast32, pSel->u32Limit));
5986 if (iSegReg == X86_SREG_SS)
5987 hmR0VmxSetPendingXcptSS(pVCpu, 0);
5988 else
5989 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5990 return VINF_HM_PENDING_XCPT;
5991 }
5992 }
5993 else
5994 {
5995 /* Check limits if it's an expand-down data segment.
5996 Note! The upper boundary is defined by the B bit, not the G bit! */
5997 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5998 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5999 {
6000 Log4Func(("Expand-down data segment limit exceeded."
6001 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6002 GCPtrLast32, pSel->u32Limit));
6003 if (iSegReg == X86_SREG_SS)
6004 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6005 else
6006 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6007 return VINF_HM_PENDING_XCPT;
6008 }
6009 }
6010 }
6011 else
6012 {
6013 /* Check permissions for the code segment. */
6014 if ( fIsWrite
6015 || !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ))
6016 {
6017 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6018 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6019 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6020 return VINF_HM_PENDING_XCPT;
6021 }
6022
6023 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6024 if ( GCPtrFirst32 > pSel->u32Limit
6025 || GCPtrLast32 > pSel->u32Limit)
6026 {
6027 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6028 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6029 if (iSegReg == X86_SREG_SS)
6030 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6031 else
6032 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6033 return VINF_HM_PENDING_XCPT;
6034 }
6035 }
6036 }
6037 else
6038 {
6039 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6040 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6041 return VINF_HM_PENDING_XCPT;
6042 }
6043
6044 *pGCPtrMem = GCPtrMem;
6045 return VINF_SUCCESS;
6046}
6047
6048
6049/**
6050 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6051 * guest attempting to execute a VMX instruction.
6052 *
6053 * @returns Strict VBox status code (i.e. informational status codes too).
6054 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6055 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
6056 *
6057 * @param pVCpu The cross context virtual CPU structure.
6058 * @param pVmxTransient Pointer to the VMX transient structure.
6059 *
6060 * @todo NstVmx: Document other error codes when VM-exit is implemented.
6061 * @remarks No-long-jump zone!!!
6062 */
6063static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6064{
6065 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
6066 | CPUMCTX_EXTRN_HWVIRT);
6067
6068 if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6069 || ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6070 && !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6071 {
6072 Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6073 hmR0VmxSetPendingXcptUD(pVCpu);
6074 return VINF_HM_PENDING_XCPT;
6075 }
6076
6077 if (pVmxTransient->uExitReason == VMX_EXIT_VMXON)
6078 {
6079 /*
6080 * We check CR4.VMXE because it is required to be always set while in VMX operation
6081 * by physical CPUs and our CR4 read shadow is only consulted when executing specific
6082 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6083 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6084 */
6085 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6086 {
6087 Log4Func(("CR4.VMXE is not set -> #UD\n"));
6088 hmR0VmxSetPendingXcptUD(pVCpu);
6089 return VINF_HM_PENDING_XCPT;
6090 }
6091 }
6092 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6093 {
6094 /*
6095 * The guest has not entered VMX operation but attempted to execute a VMX instruction
6096 * (other than VMXON), we need to raise a #UD.
6097 */
6098 Log4Func(("Not in VMX root mode -> #UD\n"));
6099 hmR0VmxSetPendingXcptUD(pVCpu);
6100 return VINF_HM_PENDING_XCPT;
6101 }
6102
6103 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6104 {
6105 /*
6106 * The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6107 * the guest hypervisor deal with it.
6108 */
6109 /** @todo NSTVMX: Trigger a VM-exit */
6110 }
6111
6112 /*
6113 * VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6114 * (above) takes preceedence over the CPL check.
6115 */
6116 if (CPUMGetGuestCPL(pVCpu) > 0)
6117 {
6118 Log4Func(("CPL > 0 -> #GP(0)\n"));
6119 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6120 return VINF_HM_PENDING_XCPT;
6121 }
6122
6123 return VINF_SUCCESS;
6124}
6125
6126#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6127
6128
6129/**
6130 * Handle a condition that occurred while delivering an event through the guest
6131 * IDT.
6132 *
6133 * @returns Strict VBox status code (i.e. informational status codes too).
6134 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6135 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6136 * to continue execution of the guest which will delivery the \#DF.
6137 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6138 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6139 *
6140 * @param pVCpu The cross context virtual CPU structure.
6141 * @param pVmxTransient Pointer to the VMX transient structure.
6142 *
6143 * @remarks No-long-jump zone!!!
6144 */
6145static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6146{
6147 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6148
6149 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6150 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6151 AssertRCReturn(rc2, rc2);
6152
6153 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6154 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6155 {
6156 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6157 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6158
6159 /*
6160 * If the event was a software interrupt (generated with INT n) or a software exception
6161 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6162 * can handle the VM-exit and continue guest execution which will re-execute the
6163 * instruction rather than re-injecting the exception, as that can cause premature
6164 * trips to ring-3 before injection and involve TRPM which currently has no way of
6165 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6166 * the problem).
6167 */
6168 IEMXCPTRAISE enmRaise;
6169 IEMXCPTRAISEINFO fRaiseInfo;
6170 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6171 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6172 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6173 {
6174 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6175 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6176 }
6177 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6178 {
6179 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6180 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6181 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6182 /** @todo Make AssertMsgReturn as just AssertMsg later. */
6183 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6184 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6185 uExitVectorType), VERR_VMX_IPE_5);
6186
6187 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6188
6189 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6190 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6191 {
6192 pVmxTransient->fVectoringPF = true;
6193 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6194 }
6195 }
6196 else
6197 {
6198 /*
6199 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6200 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6201 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
6202 */
6203 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6204 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6205 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6206 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6207 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6208 }
6209
6210 /*
6211 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6212 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6213 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6214 * subsequent VM-entry would fail.
6215 *
6216 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6217 */
6218 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6219 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6220 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6221 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6222 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6223 {
6224 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6225 }
6226
6227 switch (enmRaise)
6228 {
6229 case IEMXCPTRAISE_CURRENT_XCPT:
6230 {
6231 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6232 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6233 Assert(rcStrict == VINF_SUCCESS);
6234 break;
6235 }
6236
6237 case IEMXCPTRAISE_PREV_EVENT:
6238 {
6239 uint32_t u32ErrCode;
6240 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6241 {
6242 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6243 AssertRCReturn(rc2, rc2);
6244 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6245 }
6246 else
6247 u32ErrCode = 0;
6248
6249 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6250 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6251 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6252 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6253
6254 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6255 pVCpu->hm.s.Event.u32ErrCode));
6256 Assert(rcStrict == VINF_SUCCESS);
6257 break;
6258 }
6259
6260 case IEMXCPTRAISE_REEXEC_INSTR:
6261 Assert(rcStrict == VINF_SUCCESS);
6262 break;
6263
6264 case IEMXCPTRAISE_DOUBLE_FAULT:
6265 {
6266 /*
6267 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6268 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6269 */
6270 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6271 {
6272 pVmxTransient->fVectoringDoublePF = true;
6273 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6274 pVCpu->cpum.GstCtx.cr2));
6275 rcStrict = VINF_SUCCESS;
6276 }
6277 else
6278 {
6279 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6280 hmR0VmxSetPendingXcptDF(pVCpu);
6281 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6282 uIdtVector, uExitVector));
6283 rcStrict = VINF_HM_DOUBLE_FAULT;
6284 }
6285 break;
6286 }
6287
6288 case IEMXCPTRAISE_TRIPLE_FAULT:
6289 {
6290 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6291 rcStrict = VINF_EM_RESET;
6292 break;
6293 }
6294
6295 case IEMXCPTRAISE_CPU_HANG:
6296 {
6297 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6298 rcStrict = VERR_EM_GUEST_CPU_HANG;
6299 break;
6300 }
6301
6302 default:
6303 {
6304 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6305 rcStrict = VERR_VMX_IPE_2;
6306 break;
6307 }
6308 }
6309 }
6310 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6311 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6312 && uExitVector != X86_XCPT_DF
6313 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6314 {
6315 /*
6316 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6317 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6318 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6319 */
6320 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6321 {
6322 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6323 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6324 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6325 }
6326 }
6327
6328 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6329 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6330 return rcStrict;
6331}
6332
6333
6334/**
6335 * Imports a guest segment register from the current VMCS into
6336 * the guest-CPU context.
6337 *
6338 * @returns VBox status code.
6339 * @param pVCpu The cross context virtual CPU structure.
6340 * @param idxSel Index of the selector in the VMCS.
6341 * @param idxLimit Index of the segment limit in the VMCS.
6342 * @param idxBase Index of the segment base in the VMCS.
6343 * @param idxAccess Index of the access rights of the segment in the VMCS.
6344 * @param pSelReg Pointer to the segment selector.
6345 *
6346 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6347 * do not log!
6348 *
6349 * @remarks Never call this function directly!!! Use the
6350 * HMVMX_IMPORT_SREG() macro as that takes care
6351 * of whether to read from the VMCS cache or not.
6352 */
6353static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6354 PCPUMSELREG pSelReg)
6355{
6356 NOREF(pVCpu);
6357
6358 uint32_t u32Sel;
6359 uint32_t u32Limit;
6360 uint32_t u32Attr;
6361 uint64_t u64Base;
6362 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6363 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6364 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6365 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6366 AssertRCReturn(rc, rc);
6367
6368 pSelReg->Sel = (uint16_t)u32Sel;
6369 pSelReg->ValidSel = (uint16_t)u32Sel;
6370 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6371 pSelReg->u32Limit = u32Limit;
6372 pSelReg->u64Base = u64Base;
6373 pSelReg->Attr.u = u32Attr;
6374
6375 /*
6376 * If VT-x marks the segment as unusable, most other bits remain undefined:
6377 * - For CS the L, D and G bits have meaning.
6378 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6379 * - For the remaining data segments no bits are defined.
6380 *
6381 * The present bit and the unusable bit has been observed to be set at the
6382 * same time (the selector was supposed to be invalid as we started executing
6383 * a V8086 interrupt in ring-0).
6384 *
6385 * What should be important for the rest of the VBox code, is that the P bit is
6386 * cleared. Some of the other VBox code recognizes the unusable bit, but
6387 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6388 * safe side here, we'll strip off P and other bits we don't care about. If
6389 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6390 *
6391 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6392 */
6393 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6394 {
6395 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6396
6397 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6398 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6399 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6400#ifdef VBOX_STRICT
6401 VMMRZCallRing3Disable(pVCpu);
6402 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6403# ifdef DEBUG_bird
6404 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6405 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6406 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6407# endif
6408 VMMRZCallRing3Enable(pVCpu);
6409#endif
6410 }
6411 return VINF_SUCCESS;
6412}
6413
6414
6415/**
6416 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6417 *
6418 * @returns VBox status code.
6419 * @param pVCpu The cross context virtual CPU structure.
6420 *
6421 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6422 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6423 * instead!!!
6424 */
6425DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6426{
6427 uint64_t u64Val;
6428 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6429 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6430 {
6431 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6432 if (RT_SUCCESS(rc))
6433 {
6434 pCtx->rip = u64Val;
6435 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6436 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6437 }
6438 return rc;
6439 }
6440 return VINF_SUCCESS;
6441}
6442
6443
6444/**
6445 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6446 *
6447 * @returns VBox status code.
6448 * @param pVCpu The cross context virtual CPU structure.
6449 *
6450 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6451 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6452 * instead!!!
6453 */
6454DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6455{
6456 uint32_t u32Val;
6457 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6458 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6459 {
6460 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6461 if (RT_SUCCESS(rc))
6462 {
6463 pCtx->eflags.u32 = u32Val;
6464
6465 /* Restore eflags for real-on-v86-mode hack. */
6466 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6467 {
6468 pCtx->eflags.Bits.u1VM = 0;
6469 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6470 }
6471 }
6472 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6473 return rc;
6474 }
6475 return VINF_SUCCESS;
6476}
6477
6478
6479/**
6480 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6481 * context.
6482 *
6483 * @returns VBox status code.
6484 * @param pVCpu The cross context virtual CPU structure.
6485 *
6486 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6487 * do not log!
6488 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6489 * instead!!!
6490 */
6491DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6492{
6493 uint32_t u32Val;
6494 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6495 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6496 if (RT_SUCCESS(rc))
6497 {
6498 /*
6499 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6500 * might need them in hmR0VmxEvaluatePendingEvent().
6501 */
6502 if (!u32Val)
6503 {
6504 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6505 {
6506 rc = hmR0VmxImportGuestRip(pVCpu);
6507 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6508 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6509 }
6510
6511 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6512 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6513 }
6514 else
6515 {
6516 rc = hmR0VmxImportGuestRip(pVCpu);
6517 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6518
6519 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6520 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6521 {
6522 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6523 }
6524 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6525 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6526
6527 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6528 {
6529 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6530 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6531 }
6532 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6533 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6534 }
6535 }
6536 return rc;
6537}
6538
6539
6540/**
6541 * Worker for VMXR0ImportStateOnDemand.
6542 *
6543 * @returns VBox status code.
6544 * @param pVCpu The cross context virtual CPU structure.
6545 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6546 */
6547static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6548{
6549#define VMXLOCAL_BREAK_RC(a_rc) \
6550 if (RT_FAILURE(a_rc)) \
6551 break
6552
6553 int rc = VINF_SUCCESS;
6554 PVM pVM = pVCpu->CTX_SUFF(pVM);
6555 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6556 uint64_t u64Val;
6557 uint32_t u32Val;
6558
6559 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6560 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6561
6562 /*
6563 * We disable interrupts to make the updating of the state and in particular
6564 * the fExtrn modification atomic wrt to preemption hooks.
6565 */
6566 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6567
6568 fWhat &= pCtx->fExtrn;
6569 if (fWhat)
6570 {
6571 do
6572 {
6573 if (fWhat & CPUMCTX_EXTRN_RIP)
6574 {
6575 rc = hmR0VmxImportGuestRip(pVCpu);
6576 VMXLOCAL_BREAK_RC(rc);
6577 }
6578
6579 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6580 {
6581 rc = hmR0VmxImportGuestRFlags(pVCpu);
6582 VMXLOCAL_BREAK_RC(rc);
6583 }
6584
6585 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6586 {
6587 rc = hmR0VmxImportGuestIntrState(pVCpu);
6588 VMXLOCAL_BREAK_RC(rc);
6589 }
6590
6591 if (fWhat & CPUMCTX_EXTRN_RSP)
6592 {
6593 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6594 VMXLOCAL_BREAK_RC(rc);
6595 pCtx->rsp = u64Val;
6596 }
6597
6598 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6599 {
6600 if (fWhat & CPUMCTX_EXTRN_CS)
6601 {
6602 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6603 rc |= hmR0VmxImportGuestRip(pVCpu);
6604 VMXLOCAL_BREAK_RC(rc);
6605 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6606 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6607 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6608 }
6609 if (fWhat & CPUMCTX_EXTRN_SS)
6610 {
6611 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6612 VMXLOCAL_BREAK_RC(rc);
6613 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6614 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6615 }
6616 if (fWhat & CPUMCTX_EXTRN_DS)
6617 {
6618 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6619 VMXLOCAL_BREAK_RC(rc);
6620 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6621 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6622 }
6623 if (fWhat & CPUMCTX_EXTRN_ES)
6624 {
6625 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6626 VMXLOCAL_BREAK_RC(rc);
6627 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6628 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6629 }
6630 if (fWhat & CPUMCTX_EXTRN_FS)
6631 {
6632 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6633 VMXLOCAL_BREAK_RC(rc);
6634 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6635 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6636 }
6637 if (fWhat & CPUMCTX_EXTRN_GS)
6638 {
6639 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6640 VMXLOCAL_BREAK_RC(rc);
6641 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6642 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6643 }
6644 }
6645
6646 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6647 {
6648 if (fWhat & CPUMCTX_EXTRN_LDTR)
6649 {
6650 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6651 VMXLOCAL_BREAK_RC(rc);
6652 }
6653
6654 if (fWhat & CPUMCTX_EXTRN_GDTR)
6655 {
6656 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6657 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6658 VMXLOCAL_BREAK_RC(rc);
6659 pCtx->gdtr.pGdt = u64Val;
6660 pCtx->gdtr.cbGdt = u32Val;
6661 }
6662
6663 /* Guest IDTR. */
6664 if (fWhat & CPUMCTX_EXTRN_IDTR)
6665 {
6666 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6667 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6668 VMXLOCAL_BREAK_RC(rc);
6669 pCtx->idtr.pIdt = u64Val;
6670 pCtx->idtr.cbIdt = u32Val;
6671 }
6672
6673 /* Guest TR. */
6674 if (fWhat & CPUMCTX_EXTRN_TR)
6675 {
6676 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6677 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6678 {
6679 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6680 VMXLOCAL_BREAK_RC(rc);
6681 }
6682 }
6683 }
6684
6685 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6686 {
6687 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6688 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6689 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6690 pCtx->SysEnter.cs = u32Val;
6691 VMXLOCAL_BREAK_RC(rc);
6692 }
6693
6694#if HC_ARCH_BITS == 64
6695 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6696 {
6697 if ( pVM->hm.s.fAllow64BitGuests
6698 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6699 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6700 }
6701
6702 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6703 {
6704 if ( pVM->hm.s.fAllow64BitGuests
6705 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6706 {
6707 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6708 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6709 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6710 }
6711 }
6712#endif
6713
6714 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6715#if HC_ARCH_BITS == 32
6716 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6717#endif
6718 )
6719 {
6720 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6721 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6722 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6723 {
6724 switch (pMsr->u32Msr)
6725 {
6726#if HC_ARCH_BITS == 32
6727 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6728 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6729 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6730 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6731#endif
6732 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6733 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6734 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6735 default:
6736 {
6737 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6738 ASMSetFlags(fEFlags);
6739 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6740 cMsrs));
6741 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6742 }
6743 }
6744 }
6745 }
6746
6747 if (fWhat & CPUMCTX_EXTRN_DR7)
6748 {
6749 if (!pVCpu->hm.s.fUsingHyperDR7)
6750 {
6751 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6752 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6753 VMXLOCAL_BREAK_RC(rc);
6754 pCtx->dr[7] = u32Val;
6755 }
6756 }
6757
6758 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6759 {
6760 uint32_t u32Shadow;
6761 if (fWhat & CPUMCTX_EXTRN_CR0)
6762 {
6763 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6764 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6765 VMXLOCAL_BREAK_RC(rc);
6766 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6767 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6768 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6769 CPUMSetGuestCR0(pVCpu, u32Val);
6770 VMMRZCallRing3Enable(pVCpu);
6771 }
6772
6773 if (fWhat & CPUMCTX_EXTRN_CR4)
6774 {
6775 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6776 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6777 VMXLOCAL_BREAK_RC(rc);
6778 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6779 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6780 CPUMSetGuestCR4(pVCpu, u32Val);
6781 }
6782
6783 if (fWhat & CPUMCTX_EXTRN_CR3)
6784 {
6785 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6786 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6787 || ( pVM->hm.s.fNestedPaging
6788 && CPUMIsGuestPagingEnabledEx(pCtx)))
6789 {
6790 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6791 if (pCtx->cr3 != u64Val)
6792 {
6793 CPUMSetGuestCR3(pVCpu, u64Val);
6794 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6795 }
6796
6797 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6798 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6799 if (CPUMIsGuestInPAEModeEx(pCtx))
6800 {
6801 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6802 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6803 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6804 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6805 VMXLOCAL_BREAK_RC(rc);
6806 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6807 }
6808 }
6809 }
6810 }
6811 } while (0);
6812
6813 if (RT_SUCCESS(rc))
6814 {
6815 /* Update fExtrn. */
6816 pCtx->fExtrn &= ~fWhat;
6817
6818 /* If everything has been imported, clear the HM keeper bit. */
6819 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6820 {
6821 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6822 Assert(!pCtx->fExtrn);
6823 }
6824 }
6825 }
6826 else
6827 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6828
6829 ASMSetFlags(fEFlags);
6830
6831 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6832
6833 /*
6834 * Honor any pending CR3 updates.
6835 *
6836 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6837 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6838 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6839 *
6840 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6841 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6842 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6843 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6844 *
6845 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6846 */
6847 if (VMMRZCallRing3IsEnabled(pVCpu))
6848 {
6849 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6850 {
6851 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6852 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6853 }
6854
6855 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6856 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6857
6858 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6859 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6860 }
6861
6862 return VINF_SUCCESS;
6863#undef VMXLOCAL_BREAK_RC
6864}
6865
6866
6867/**
6868 * Saves the guest state from the VMCS into the guest-CPU context.
6869 *
6870 * @returns VBox status code.
6871 * @param pVCpu The cross context virtual CPU structure.
6872 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6873 */
6874VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6875{
6876 return hmR0VmxImportGuestState(pVCpu, fWhat);
6877}
6878
6879
6880/**
6881 * Check per-VM and per-VCPU force flag actions that require us to go back to
6882 * ring-3 for one reason or another.
6883 *
6884 * @returns Strict VBox status code (i.e. informational status codes too)
6885 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6886 * ring-3.
6887 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6888 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6889 * interrupts)
6890 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6891 * all EMTs to be in ring-3.
6892 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6893 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6894 * to the EM loop.
6895 *
6896 * @param pVCpu The cross context virtual CPU structure.
6897 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6898 */
6899static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6900{
6901 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6902
6903 /*
6904 * Anything pending? Should be more likely than not if we're doing a good job.
6905 */
6906 PVM pVM = pVCpu->CTX_SUFF(pVM);
6907 if ( !fStepping
6908 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6909 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6910 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6911 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6912 return VINF_SUCCESS;
6913
6914 /* Pending PGM C3 sync. */
6915 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6916 {
6917 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6918 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6919 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6920 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6921 if (rcStrict2 != VINF_SUCCESS)
6922 {
6923 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6924 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6925 return rcStrict2;
6926 }
6927 }
6928
6929 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6930 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6931 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6932 {
6933 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6934 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6935 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6936 return rc2;
6937 }
6938
6939 /* Pending VM request packets, such as hardware interrupts. */
6940 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6941 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6942 {
6943 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6944 return VINF_EM_PENDING_REQUEST;
6945 }
6946
6947 /* Pending PGM pool flushes. */
6948 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6949 {
6950 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6951 return VINF_PGM_POOL_FLUSH_PENDING;
6952 }
6953
6954 /* Pending DMA requests. */
6955 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6956 {
6957 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6958 return VINF_EM_RAW_TO_R3;
6959 }
6960
6961 return VINF_SUCCESS;
6962}
6963
6964
6965/**
6966 * Converts any TRPM trap into a pending HM event. This is typically used when
6967 * entering from ring-3 (not longjmp returns).
6968 *
6969 * @param pVCpu The cross context virtual CPU structure.
6970 */
6971static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6972{
6973 Assert(TRPMHasTrap(pVCpu));
6974 Assert(!pVCpu->hm.s.Event.fPending);
6975
6976 uint8_t uVector;
6977 TRPMEVENT enmTrpmEvent;
6978 RTGCUINT uErrCode;
6979 RTGCUINTPTR GCPtrFaultAddress;
6980 uint8_t cbInstr;
6981
6982 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6983 AssertRC(rc);
6984
6985 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6986 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
6987 if (enmTrpmEvent == TRPM_TRAP)
6988 {
6989 switch (uVector)
6990 {
6991 case X86_XCPT_NMI:
6992 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6993 break;
6994
6995 case X86_XCPT_BP:
6996 case X86_XCPT_OF:
6997 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6998 break;
6999
7000 case X86_XCPT_PF:
7001 case X86_XCPT_DF:
7002 case X86_XCPT_TS:
7003 case X86_XCPT_NP:
7004 case X86_XCPT_SS:
7005 case X86_XCPT_GP:
7006 case X86_XCPT_AC:
7007 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7008 RT_FALL_THRU();
7009 default:
7010 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7011 break;
7012 }
7013 }
7014 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7015 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7016 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7017 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7018 else
7019 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7020
7021 rc = TRPMResetTrap(pVCpu);
7022 AssertRC(rc);
7023 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7024 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7025
7026 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7027}
7028
7029
7030/**
7031 * Converts the pending HM event into a TRPM trap.
7032 *
7033 * @param pVCpu The cross context virtual CPU structure.
7034 */
7035static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7036{
7037 Assert(pVCpu->hm.s.Event.fPending);
7038
7039 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7040 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7041 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7042 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7043
7044 /* If a trap was already pending, we did something wrong! */
7045 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7046
7047 TRPMEVENT enmTrapType;
7048 switch (uVectorType)
7049 {
7050 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7051 enmTrapType = TRPM_HARDWARE_INT;
7052 break;
7053
7054 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7055 enmTrapType = TRPM_SOFTWARE_INT;
7056 break;
7057
7058 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7059 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7060 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7061 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7062 enmTrapType = TRPM_TRAP;
7063 break;
7064
7065 default:
7066 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7067 enmTrapType = TRPM_32BIT_HACK;
7068 break;
7069 }
7070
7071 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7072
7073 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7074 AssertRC(rc);
7075
7076 if (fErrorCodeValid)
7077 TRPMSetErrorCode(pVCpu, uErrorCode);
7078
7079 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7080 && uVector == X86_XCPT_PF)
7081 {
7082 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7083 }
7084 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7085 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7086 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7087 {
7088 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7089 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7090 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7091 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7092 }
7093
7094 /* Clear any pending events from the VMCS. */
7095 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7096 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
7097
7098 /* We're now done converting the pending event. */
7099 pVCpu->hm.s.Event.fPending = false;
7100}
7101
7102
7103/**
7104 * Does the necessary state syncing before returning to ring-3 for any reason
7105 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7106 *
7107 * @returns VBox status code.
7108 * @param pVCpu The cross context virtual CPU structure.
7109 * @param fImportState Whether to import the guest state from the VMCS back
7110 * to the guest-CPU context.
7111 *
7112 * @remarks No-long-jmp zone!!!
7113 */
7114static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7115{
7116 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7117 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7118
7119 RTCPUID idCpu = RTMpCpuId();
7120 Log4Func(("HostCpuId=%u\n", idCpu));
7121
7122 /*
7123 * !!! IMPORTANT !!!
7124 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7125 */
7126
7127 /* Save the guest state if necessary. */
7128 if (fImportState)
7129 {
7130 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7131 AssertRCReturn(rc, rc);
7132 }
7133
7134 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7135 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7136 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7137
7138 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7139#ifdef VBOX_STRICT
7140 if (CPUMIsHyperDebugStateActive(pVCpu))
7141 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7142#endif
7143 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7144 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7145 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7146
7147#if HC_ARCH_BITS == 64
7148 /* Restore host-state bits that VT-x only restores partially. */
7149 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7150 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7151 {
7152 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7153 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7154 }
7155 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7156#endif
7157
7158 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7159 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7160 {
7161 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7162 if (!fImportState)
7163 {
7164 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
7165 AssertRCReturn(rc, rc);
7166 }
7167 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7168 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7169 }
7170 else
7171 pVCpu->hm.s.vmx.fLazyMsrs = 0;
7172
7173 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7174 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7175
7176 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7177 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7178 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7179 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7180 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7181 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7182 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7183 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7184 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7185
7186 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7187
7188 /** @todo This partially defeats the purpose of having preemption hooks.
7189 * The problem is, deregistering the hooks should be moved to a place that
7190 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7191 * context.
7192 */
7193 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7194 {
7195 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7196 AssertRCReturn(rc, rc);
7197
7198 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7199 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7200 }
7201 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7202 NOREF(idCpu);
7203
7204 return VINF_SUCCESS;
7205}
7206
7207
7208/**
7209 * Leaves the VT-x session.
7210 *
7211 * @returns VBox status code.
7212 * @param pVCpu The cross context virtual CPU structure.
7213 *
7214 * @remarks No-long-jmp zone!!!
7215 */
7216static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7217{
7218 HM_DISABLE_PREEMPT(pVCpu);
7219 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7220 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7221 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7222
7223 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7224 and done this from the VMXR0ThreadCtxCallback(). */
7225 if (!pVCpu->hm.s.fLeaveDone)
7226 {
7227 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7228 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7229 pVCpu->hm.s.fLeaveDone = true;
7230 }
7231 Assert(!pVCpu->cpum.GstCtx.fExtrn);
7232
7233 /*
7234 * !!! IMPORTANT !!!
7235 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7236 */
7237
7238 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7239 /** @todo Deregistering here means we need to VMCLEAR always
7240 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7241 * for calling VMMR0ThreadCtxHookDisable here! */
7242 VMMR0ThreadCtxHookDisable(pVCpu);
7243
7244 /* Leave HM context. This takes care of local init (term). */
7245 int rc = HMR0LeaveCpu(pVCpu);
7246
7247 HM_RESTORE_PREEMPT();
7248 return rc;
7249}
7250
7251
7252/**
7253 * Does the necessary state syncing before doing a longjmp to ring-3.
7254 *
7255 * @returns VBox status code.
7256 * @param pVCpu The cross context virtual CPU structure.
7257 *
7258 * @remarks No-long-jmp zone!!!
7259 */
7260DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7261{
7262 return hmR0VmxLeaveSession(pVCpu);
7263}
7264
7265
7266/**
7267 * Take necessary actions before going back to ring-3.
7268 *
7269 * An action requires us to go back to ring-3. This function does the necessary
7270 * steps before we can safely return to ring-3. This is not the same as longjmps
7271 * to ring-3, this is voluntary and prepares the guest so it may continue
7272 * executing outside HM (recompiler/IEM).
7273 *
7274 * @returns VBox status code.
7275 * @param pVCpu The cross context virtual CPU structure.
7276 * @param rcExit The reason for exiting to ring-3. Can be
7277 * VINF_VMM_UNKNOWN_RING3_CALL.
7278 */
7279static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7280{
7281 Assert(pVCpu);
7282 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7283
7284 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7285 {
7286 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7287 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7288 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7289 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7290 }
7291
7292 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7293 VMMRZCallRing3Disable(pVCpu);
7294 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7295
7296 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7297 if (pVCpu->hm.s.Event.fPending)
7298 {
7299 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7300 Assert(!pVCpu->hm.s.Event.fPending);
7301 }
7302
7303 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7304 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7305
7306 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7307 and if we're injecting an event we should have a TRPM trap pending. */
7308 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7309#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7310 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7311#endif
7312
7313 /* Save guest state and restore host state bits. */
7314 int rc = hmR0VmxLeaveSession(pVCpu);
7315 AssertRCReturn(rc, rc);
7316 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7317 /* Thread-context hooks are unregistered at this point!!! */
7318
7319 /* Sync recompiler state. */
7320 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7321 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7322 | CPUM_CHANGED_LDTR
7323 | CPUM_CHANGED_GDTR
7324 | CPUM_CHANGED_IDTR
7325 | CPUM_CHANGED_TR
7326 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7327 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7328 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7329 {
7330 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7331 }
7332
7333 Assert(!pVCpu->hm.s.fClearTrapFlag);
7334
7335 /* Update the exit-to-ring 3 reason. */
7336 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7337
7338 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7339 if (rcExit != VINF_EM_RAW_INTERRUPT)
7340 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7341
7342 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7343
7344 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7345 VMMRZCallRing3RemoveNotification(pVCpu);
7346 VMMRZCallRing3Enable(pVCpu);
7347
7348 return rc;
7349}
7350
7351
7352/**
7353 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7354 * longjump to ring-3 and possibly get preempted.
7355 *
7356 * @returns VBox status code.
7357 * @param pVCpu The cross context virtual CPU structure.
7358 * @param enmOperation The operation causing the ring-3 longjump.
7359 * @param pvUser User argument, currently unused, NULL.
7360 */
7361static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7362{
7363 RT_NOREF(pvUser);
7364 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7365 {
7366 /*
7367 * !!! IMPORTANT !!!
7368 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7369 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7370 */
7371 VMMRZCallRing3RemoveNotification(pVCpu);
7372 VMMRZCallRing3Disable(pVCpu);
7373 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7374 RTThreadPreemptDisable(&PreemptState);
7375
7376 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7377 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7378 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7379
7380#if HC_ARCH_BITS == 64
7381 /* Restore host-state bits that VT-x only restores partially. */
7382 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7383 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7384 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7385 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7386#endif
7387
7388 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7389 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7390 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7391
7392 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7393 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7394 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7395 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7396 {
7397 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7398 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7399 }
7400
7401 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7402 VMMR0ThreadCtxHookDisable(pVCpu);
7403 HMR0LeaveCpu(pVCpu);
7404 RTThreadPreemptRestore(&PreemptState);
7405 return VINF_SUCCESS;
7406 }
7407
7408 Assert(pVCpu);
7409 Assert(pvUser);
7410 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7411 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7412
7413 VMMRZCallRing3Disable(pVCpu);
7414 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7415
7416 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7417
7418 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7419 AssertRCReturn(rc, rc);
7420
7421 VMMRZCallRing3Enable(pVCpu);
7422 return VINF_SUCCESS;
7423}
7424
7425
7426/**
7427 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7428 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7429 *
7430 * @param pVCpu The cross context virtual CPU structure.
7431 */
7432DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7433{
7434 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7435 {
7436 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7437 {
7438 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7439 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7440 AssertRC(rc);
7441 Log4Func(("Setup interrupt-window exiting\n"));
7442 }
7443 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7444}
7445
7446
7447/**
7448 * Clears the interrupt-window exiting control in the VMCS.
7449 *
7450 * @param pVCpu The cross context virtual CPU structure.
7451 */
7452DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7453{
7454 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7455 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7456 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7457 AssertRC(rc);
7458 Log4Func(("Cleared interrupt-window exiting\n"));
7459}
7460
7461
7462/**
7463 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7464 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7465 *
7466 * @param pVCpu The cross context virtual CPU structure.
7467 */
7468DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7469{
7470 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7471 {
7472 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7473 {
7474 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7475 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7476 AssertRC(rc);
7477 Log4Func(("Setup NMI-window exiting\n"));
7478 }
7479 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7480}
7481
7482
7483/**
7484 * Clears the NMI-window exiting control in the VMCS.
7485 *
7486 * @param pVCpu The cross context virtual CPU structure.
7487 */
7488DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7489{
7490 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7491 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7492 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7493 AssertRC(rc);
7494 Log4Func(("Cleared NMI-window exiting\n"));
7495}
7496
7497
7498/**
7499 * Evaluates the event to be delivered to the guest and sets it as the pending
7500 * event.
7501 *
7502 * @returns The VT-x guest-interruptibility state.
7503 * @param pVCpu The cross context virtual CPU structure.
7504 */
7505static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7506{
7507 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7508 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7509 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7510 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7511 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7512 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7513
7514 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7515 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7516 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7517 Assert(!TRPMHasTrap(pVCpu));
7518
7519 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7520 APICUpdatePendingInterrupts(pVCpu);
7521
7522 /*
7523 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7524 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7525 */
7526 /** @todo SMI. SMIs take priority over NMIs. */
7527 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7528 {
7529 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7530 if ( !pVCpu->hm.s.Event.fPending
7531 && !fBlockNmi
7532 && !fBlockSti
7533 && !fBlockMovSS)
7534 {
7535 Log4Func(("Pending NMI\n"));
7536 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7537 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7538
7539 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7540 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7541 }
7542 else
7543 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7544 }
7545 /*
7546 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7547 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7548 */
7549 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7550 && !pVCpu->hm.s.fSingleInstruction)
7551 {
7552 Assert(!DBGFIsStepping(pVCpu));
7553 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7554 AssertRCReturn(rc, 0);
7555 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7556 if ( !pVCpu->hm.s.Event.fPending
7557 && !fBlockInt
7558 && !fBlockSti
7559 && !fBlockMovSS)
7560 {
7561 uint8_t u8Interrupt;
7562 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7563 if (RT_SUCCESS(rc))
7564 {
7565 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7566 uint32_t u32IntInfo = u8Interrupt
7567 | VMX_EXIT_INT_INFO_VALID
7568 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7569
7570 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7571 }
7572 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7573 {
7574 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7575 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7576 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7577
7578 /*
7579 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7580 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7581 * need to re-set this force-flag here.
7582 */
7583 }
7584 else
7585 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7586 }
7587 else
7588 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7589 }
7590
7591 return fIntrState;
7592}
7593
7594
7595/**
7596 * Sets a pending-debug exception to be delivered to the guest if the guest is
7597 * single-stepping in the VMCS.
7598 *
7599 * @param pVCpu The cross context virtual CPU structure.
7600 */
7601DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7602{
7603 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7604 RT_NOREF(pVCpu);
7605 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7606}
7607
7608
7609/**
7610 * Injects any pending events into the guest if the guest is in a state to
7611 * receive them.
7612 *
7613 * @returns Strict VBox status code (i.e. informational status codes too).
7614 * @param pVCpu The cross context virtual CPU structure.
7615 * @param fIntrState The VT-x guest-interruptibility state.
7616 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7617 * return VINF_EM_DBG_STEPPED if the event was
7618 * dispatched directly.
7619 */
7620static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7621{
7622 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7623 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7624
7625 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7626 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7627
7628 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7629 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7630 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7631 Assert(!TRPMHasTrap(pVCpu));
7632
7633 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7634 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7635 if (pVCpu->hm.s.Event.fPending)
7636 {
7637 /*
7638 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7639 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7640 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7641 *
7642 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7643 */
7644 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7645#ifdef VBOX_STRICT
7646 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7647 {
7648 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7649 Assert(!fBlockInt);
7650 Assert(!fBlockSti);
7651 Assert(!fBlockMovSS);
7652 }
7653 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
7654 {
7655 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7656 Assert(!fBlockSti);
7657 Assert(!fBlockMovSS);
7658 Assert(!fBlockNmi);
7659 }
7660#endif
7661 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7662 uIntType));
7663 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7664 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7665 &fIntrState);
7666 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7667
7668 /* Update the interruptibility-state as it could have been changed by
7669 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7670 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7671 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7672
7673 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7674 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7675 else
7676 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7677 }
7678
7679 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7680 if ( fBlockSti
7681 || fBlockMovSS)
7682 {
7683 if (!pVCpu->hm.s.fSingleInstruction)
7684 {
7685 /*
7686 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7687 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7688 * See Intel spec. 27.3.4 "Saving Non-Register State".
7689 */
7690 Assert(!DBGFIsStepping(pVCpu));
7691 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7692 AssertRCReturn(rc, rc);
7693 if (pCtx->eflags.Bits.u1TF)
7694 {
7695 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7696 AssertRCReturn(rc2, rc2);
7697 }
7698 }
7699 else if (pCtx->eflags.Bits.u1TF)
7700 {
7701 /*
7702 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7703 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7704 */
7705 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7706 fIntrState = 0;
7707 }
7708 }
7709
7710 /*
7711 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7712 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7713 */
7714 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7715 AssertRCReturn(rc3, rc3);
7716
7717 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7718 NOREF(fBlockMovSS); NOREF(fBlockSti);
7719 return rcStrict;
7720}
7721
7722
7723/**
7724 * Injects a double-fault (\#DF) exception into the VM.
7725 *
7726 * @returns Strict VBox status code (i.e. informational status codes too).
7727 * @param pVCpu The cross context virtual CPU structure.
7728 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7729 * and should return VINF_EM_DBG_STEPPED if the event
7730 * is injected directly (register modified by us, not
7731 * by hardware on VM-entry).
7732 * @param pfIntrState Pointer to the current guest interruptibility-state.
7733 * This interruptibility-state will be updated if
7734 * necessary. This cannot not be NULL.
7735 */
7736DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7737{
7738 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7739 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7740 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7741 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7742 pfIntrState);
7743}
7744
7745
7746/**
7747 * Injects a general-protection (\#GP) fault into the VM.
7748 *
7749 * @returns Strict VBox status code (i.e. informational status codes too).
7750 * @param pVCpu The cross context virtual CPU structure.
7751 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7752 * mode, i.e. in real-mode it's not valid).
7753 * @param u32ErrorCode The error code associated with the \#GP.
7754 * @param fStepping Whether we're running in
7755 * hmR0VmxRunGuestCodeStep() and should return
7756 * VINF_EM_DBG_STEPPED if the event is injected
7757 * directly (register modified by us, not by
7758 * hardware on VM-entry).
7759 * @param pfIntrState Pointer to the current guest interruptibility-state.
7760 * This interruptibility-state will be updated if
7761 * necessary. This cannot not be NULL.
7762 */
7763DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7764 uint32_t *pfIntrState)
7765{
7766 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7767 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7768 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7769 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7770 pfIntrState);
7771}
7772
7773
7774/**
7775 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7776 *
7777 * @param pVCpu The cross context virtual CPU structure.
7778 * @param uVector The software interrupt vector number.
7779 * @param cbInstr The value of RIP that is to be pushed on the guest
7780 * stack.
7781 */
7782DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, uint16_t uVector, uint32_t cbInstr)
7783{
7784 bool const fIsSwXcpt = RT_BOOL(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7785 uint32_t const u32IntType = fIsSwXcpt ? VMX_EXIT_INT_INFO_TYPE_SW_XCPT : VMX_EXIT_INT_INFO_TYPE_SW_INT;
7786 uint32_t const u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID
7787 | (u32IntType << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7788 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7789}
7790
7791
7792/**
7793 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7794 * stack.
7795 *
7796 * @returns Strict VBox status code (i.e. informational status codes too).
7797 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7798 * @param pVCpu The cross context virtual CPU structure.
7799 * @param uValue The value to push to the guest stack.
7800 */
7801static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7802{
7803 /*
7804 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7805 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7806 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7807 */
7808 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7809 if (pCtx->sp == 1)
7810 return VINF_EM_RESET;
7811 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7812 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7813 AssertRC(rc);
7814 return rc;
7815}
7816
7817
7818/**
7819 * Injects an event into the guest upon VM-entry by updating the relevant fields
7820 * in the VM-entry area in the VMCS.
7821 *
7822 * @returns Strict VBox status code (i.e. informational status codes too).
7823 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7824 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7825 *
7826 * @param pVCpu The cross context virtual CPU structure.
7827 * @param u64IntInfo The VM-entry interruption-information field.
7828 * @param cbInstr The VM-entry instruction length in bytes (for
7829 * software interrupts, exceptions and privileged
7830 * software exceptions).
7831 * @param u32ErrCode The VM-entry exception error code.
7832 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7833 * @param pfIntrState Pointer to the current guest interruptibility-state.
7834 * This interruptibility-state will be updated if
7835 * necessary. This cannot not be NULL.
7836 * @param fStepping Whether we're running in
7837 * hmR0VmxRunGuestCodeStep() and should return
7838 * VINF_EM_DBG_STEPPED if the event is injected
7839 * directly (register modified by us, not by
7840 * hardware on VM-entry).
7841 */
7842static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7843 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7844{
7845 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7846 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7847 Assert(pfIntrState);
7848
7849 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7850 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7851 uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(u32IntInfo);
7852 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(u32IntInfo);
7853
7854#ifdef VBOX_STRICT
7855 /*
7856 * Validate the error-code-valid bit for hardware exceptions.
7857 * No error codes for exceptions in real-mode.
7858 *
7859 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7860 */
7861 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7862 && !CPUMIsGuestInRealModeEx(pCtx))
7863 {
7864 switch (uVector)
7865 {
7866 case X86_XCPT_PF:
7867 case X86_XCPT_DF:
7868 case X86_XCPT_TS:
7869 case X86_XCPT_NP:
7870 case X86_XCPT_SS:
7871 case X86_XCPT_GP:
7872 case X86_XCPT_AC:
7873 AssertMsg(VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7874 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7875 RT_FALL_THRU();
7876 default:
7877 break;
7878 }
7879 }
7880#endif
7881
7882 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7883 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7884 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7885
7886 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7887
7888 /*
7889 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7890 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7891 * interrupt handler in the (real-mode) guest.
7892 *
7893 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7894 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7895 */
7896 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7897 {
7898 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7899 {
7900 /*
7901 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7902 * set the deliver-error-code bit.
7903 *
7904 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7905 */
7906 u32IntInfo &= ~VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7907 }
7908 else
7909 {
7910 PVM pVM = pVCpu->CTX_SUFF(pVM);
7911 Assert(PDMVmmDevHeapIsEnabled(pVM));
7912 Assert(pVM->hm.s.vmx.pRealModeTSS);
7913
7914 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7915 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7916 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7917 AssertRCReturn(rc2, rc2);
7918
7919 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7920 size_t const cbIdtEntry = sizeof(X86IDTR16);
7921 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7922 {
7923 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7924 if (uVector == X86_XCPT_DF)
7925 return VINF_EM_RESET;
7926
7927 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7928 if (uVector == X86_XCPT_GP)
7929 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7930
7931 /*
7932 * If we're injecting an event with no valid IDT entry, inject a #GP.
7933 * No error codes for exceptions in real-mode.
7934 *
7935 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7936 */
7937 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7938 }
7939
7940 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7941 uint16_t uGuestIp = pCtx->ip;
7942 if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
7943 {
7944 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7945 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7946 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7947 }
7948 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_INT)
7949 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7950
7951 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7952 X86IDTR16 IdtEntry;
7953 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7954 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7955 AssertRCReturn(rc2, rc2);
7956
7957 /* Construct the stack frame for the interrupt/exception handler. */
7958 VBOXSTRICTRC rcStrict;
7959 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7960 if (rcStrict == VINF_SUCCESS)
7961 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7962 if (rcStrict == VINF_SUCCESS)
7963 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7964
7965 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7966 if (rcStrict == VINF_SUCCESS)
7967 {
7968 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7969 pCtx->rip = IdtEntry.offSel;
7970 pCtx->cs.Sel = IdtEntry.uSel;
7971 pCtx->cs.ValidSel = IdtEntry.uSel;
7972 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7973 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7974 && uVector == X86_XCPT_PF)
7975 pCtx->cr2 = GCPtrFaultAddress;
7976
7977 /* If any other guest-state bits are changed here, make sure to update
7978 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7979 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7980 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7981 | HM_CHANGED_GUEST_RSP);
7982
7983 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7984 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7985 {
7986 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7987 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
7988 Log4Func(("Clearing inhibition due to STI\n"));
7989 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7990 }
7991 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7992 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7993
7994 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7995 it, if we are returning to ring-3 before executing guest code. */
7996 pVCpu->hm.s.Event.fPending = false;
7997
7998 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7999 if (fStepping)
8000 rcStrict = VINF_EM_DBG_STEPPED;
8001 }
8002 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8003 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8004 return rcStrict;
8005 }
8006 }
8007
8008 /* Validate. */
8009 Assert(VMX_EXIT_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8010 Assert(!VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8011 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8012
8013 /* Inject. */
8014 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8015 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8016 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8017 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8018 AssertRCReturn(rc, rc);
8019
8020 /* Update CR2. */
8021 if ( VMX_EXIT_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8022 && uVector == X86_XCPT_PF)
8023 pCtx->cr2 = GCPtrFaultAddress;
8024
8025 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8026
8027 return VINF_SUCCESS;
8028}
8029
8030
8031/**
8032 * Clears the interrupt-window exiting control in the VMCS and if necessary
8033 * clears the current event in the VMCS as well.
8034 *
8035 * @returns VBox status code.
8036 * @param pVCpu The cross context virtual CPU structure.
8037 *
8038 * @remarks Use this function only to clear events that have not yet been
8039 * delivered to the guest but are injected in the VMCS!
8040 * @remarks No-long-jump zone!!!
8041 */
8042static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8043{
8044 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8045 {
8046 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8047 Log4Func(("Cleared interrupt widow\n"));
8048 }
8049
8050 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8051 {
8052 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8053 Log4Func(("Cleared interrupt widow\n"));
8054 }
8055}
8056
8057
8058/**
8059 * Enters the VT-x session.
8060 *
8061 * @returns VBox status code.
8062 * @param pVCpu The cross context virtual CPU structure.
8063 * @param pHostCpu Pointer to the global CPU info struct.
8064 */
8065VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8066{
8067 AssertPtr(pVCpu);
8068 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8069 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8070 RT_NOREF(pHostCpu);
8071
8072 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8073 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8074 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8075
8076#ifdef VBOX_STRICT
8077 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8078 RTCCUINTREG uHostCR4 = ASMGetCR4();
8079 if (!(uHostCR4 & X86_CR4_VMXE))
8080 {
8081 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8082 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8083 }
8084#endif
8085
8086 /*
8087 * Load the VCPU's VMCS as the current (and active) one.
8088 */
8089 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8090 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8091 if (RT_FAILURE(rc))
8092 return rc;
8093
8094 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8095 pVCpu->hm.s.fLeaveDone = false;
8096 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8097
8098 return VINF_SUCCESS;
8099}
8100
8101
8102/**
8103 * The thread-context callback (only on platforms which support it).
8104 *
8105 * @param enmEvent The thread-context event.
8106 * @param pVCpu The cross context virtual CPU structure.
8107 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8108 * @thread EMT(pVCpu)
8109 */
8110VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8111{
8112 NOREF(fGlobalInit);
8113
8114 switch (enmEvent)
8115 {
8116 case RTTHREADCTXEVENT_OUT:
8117 {
8118 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8119 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8120 VMCPU_ASSERT_EMT(pVCpu);
8121
8122 /* No longjmps (logger flushes, locks) in this fragile context. */
8123 VMMRZCallRing3Disable(pVCpu);
8124 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8125
8126 /*
8127 * Restore host-state (FPU, debug etc.)
8128 */
8129 if (!pVCpu->hm.s.fLeaveDone)
8130 {
8131 /*
8132 * Do -not- import the guest-state here as we might already be in the middle of importing
8133 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8134 */
8135 hmR0VmxLeave(pVCpu, false /* fImportState */);
8136 pVCpu->hm.s.fLeaveDone = true;
8137 }
8138
8139 /* Leave HM context, takes care of local init (term). */
8140 int rc = HMR0LeaveCpu(pVCpu);
8141 AssertRC(rc); NOREF(rc);
8142
8143 /* Restore longjmp state. */
8144 VMMRZCallRing3Enable(pVCpu);
8145 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8146 break;
8147 }
8148
8149 case RTTHREADCTXEVENT_IN:
8150 {
8151 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8152 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8153 VMCPU_ASSERT_EMT(pVCpu);
8154
8155 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8156 VMMRZCallRing3Disable(pVCpu);
8157 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8158
8159 /* Initialize the bare minimum state required for HM. This takes care of
8160 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8161 int rc = hmR0EnterCpu(pVCpu);
8162 AssertRC(rc);
8163 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8164 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8165
8166 /* Load the active VMCS as the current one. */
8167 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8168 {
8169 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8170 AssertRC(rc); NOREF(rc);
8171 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8172 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8173 }
8174 pVCpu->hm.s.fLeaveDone = false;
8175
8176 /* Restore longjmp state. */
8177 VMMRZCallRing3Enable(pVCpu);
8178 break;
8179 }
8180
8181 default:
8182 break;
8183 }
8184}
8185
8186
8187/**
8188 * Exports the host state into the VMCS host-state area.
8189 * Sets up the VM-exit MSR-load area.
8190 *
8191 * The CPU state will be loaded from these fields on every successful VM-exit.
8192 *
8193 * @returns VBox status code.
8194 * @param pVCpu The cross context virtual CPU structure.
8195 *
8196 * @remarks No-long-jump zone!!!
8197 */
8198static int hmR0VmxExportHostState(PVMCPU pVCpu)
8199{
8200 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8201
8202 int rc = VINF_SUCCESS;
8203 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8204 {
8205 rc = hmR0VmxExportHostControlRegs();
8206 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8207
8208 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8209 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8210
8211 rc = hmR0VmxExportHostMsrs(pVCpu);
8212 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8213
8214 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8215 }
8216 return rc;
8217}
8218
8219
8220/**
8221 * Saves the host state in the VMCS host-state.
8222 *
8223 * @returns VBox status code.
8224 * @param pVCpu The cross context virtual CPU structure.
8225 *
8226 * @remarks No-long-jump zone!!!
8227 */
8228VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8229{
8230 AssertPtr(pVCpu);
8231 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8232
8233 /*
8234 * Export the host state here while entering HM context.
8235 * When thread-context hooks are used, we might get preempted and have to re-save the host
8236 * state but most of the time we won't be, so do it here before we disable interrupts.
8237 */
8238 return hmR0VmxExportHostState(pVCpu);
8239}
8240
8241
8242/**
8243 * Exports the guest state into the VMCS guest-state area.
8244 *
8245 * The will typically be done before VM-entry when the guest-CPU state and the
8246 * VMCS state may potentially be out of sync.
8247 *
8248 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8249 * VM-entry controls.
8250 * Sets up the appropriate VMX non-root function to execute guest code based on
8251 * the guest CPU mode.
8252 *
8253 * @returns VBox strict status code.
8254 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8255 * without unrestricted guest access and the VMMDev is not presently
8256 * mapped (e.g. EFI32).
8257 *
8258 * @param pVCpu The cross context virtual CPU structure.
8259 *
8260 * @remarks No-long-jump zone!!!
8261 */
8262static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8263{
8264 AssertPtr(pVCpu);
8265 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8266
8267 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8268
8269 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8270
8271 /* Determine real-on-v86 mode. */
8272 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8273 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8274 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8275 {
8276 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8277 }
8278
8279 /*
8280 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8281 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8282 */
8283 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8284 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8285
8286 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8287 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8288 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8289
8290 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8291 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8292 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8293
8294 rc = hmR0VmxExportGuestCR0(pVCpu);
8295 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8296
8297 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8298 if (rcStrict == VINF_SUCCESS)
8299 { /* likely */ }
8300 else
8301 {
8302 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8303 return rcStrict;
8304 }
8305
8306 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8307 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8308
8309 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8310 may alter controls if we determine we don't have to swap EFER after all. */
8311 rc = hmR0VmxExportGuestMsrs(pVCpu);
8312 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8313
8314 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8315 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8316
8317 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8318 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8319
8320 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8321 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8322 rc = hmR0VmxExportGuestRip(pVCpu);
8323 rc |= hmR0VmxExportGuestRsp(pVCpu);
8324 rc |= hmR0VmxExportGuestRflags(pVCpu);
8325 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8326
8327 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8328 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8329 | HM_CHANGED_GUEST_CR2
8330 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8331 | HM_CHANGED_GUEST_X87
8332 | HM_CHANGED_GUEST_SSE_AVX
8333 | HM_CHANGED_GUEST_OTHER_XSAVE
8334 | HM_CHANGED_GUEST_XCRx
8335 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8336 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8337 | HM_CHANGED_GUEST_TSC_AUX
8338 | HM_CHANGED_GUEST_OTHER_MSRS
8339 | HM_CHANGED_GUEST_HWVIRT
8340 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8341
8342 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8343 return rc;
8344}
8345
8346
8347/**
8348 * Exports the state shared between the host and guest into the VMCS.
8349 *
8350 * @param pVCpu The cross context virtual CPU structure.
8351 *
8352 * @remarks No-long-jump zone!!!
8353 */
8354static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8355{
8356 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8357 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8358
8359 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8360 {
8361 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8362 AssertRC(rc);
8363 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8364
8365 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8366 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8367 {
8368 rc = hmR0VmxExportGuestRflags(pVCpu);
8369 AssertRC(rc);
8370 }
8371 }
8372
8373 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8374 {
8375 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8376 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8377 }
8378
8379 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8380 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8381}
8382
8383
8384/**
8385 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8386 *
8387 * @returns Strict VBox status code (i.e. informational status codes too).
8388 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8389 * without unrestricted guest access and the VMMDev is not presently
8390 * mapped (e.g. EFI32).
8391 *
8392 * @param pVCpu The cross context virtual CPU structure.
8393 *
8394 * @remarks No-long-jump zone!!!
8395 */
8396static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8397{
8398 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8399 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8400 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8401
8402#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8403 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8404#endif
8405
8406 /*
8407 * For many exits it's only RIP that changes and hence try to export it first
8408 * without going through a lot of change flag checks.
8409 */
8410 VBOXSTRICTRC rcStrict;
8411 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8412 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8413 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8414 {
8415 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8416 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8417 { /* likely */}
8418 else
8419 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8420 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8421 }
8422 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8423 {
8424 rcStrict = hmR0VmxExportGuestState(pVCpu);
8425 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8426 { /* likely */}
8427 else
8428 {
8429 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8430 VBOXSTRICTRC_VAL(rcStrict)));
8431 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8432 return rcStrict;
8433 }
8434 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8435 }
8436 else
8437 rcStrict = VINF_SUCCESS;
8438
8439#ifdef VBOX_STRICT
8440 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8441 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8442 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8443 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8444 ("fCtxChanged=%#RX64\n", fCtxChanged));
8445#endif
8446 return rcStrict;
8447}
8448
8449
8450/**
8451 * Does the preparations before executing guest code in VT-x.
8452 *
8453 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8454 * recompiler/IEM. We must be cautious what we do here regarding committing
8455 * guest-state information into the VMCS assuming we assuredly execute the
8456 * guest in VT-x mode.
8457 *
8458 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8459 * the common-state (TRPM/forceflags), we must undo those changes so that the
8460 * recompiler/IEM can (and should) use them when it resumes guest execution.
8461 * Otherwise such operations must be done when we can no longer exit to ring-3.
8462 *
8463 * @returns Strict VBox status code (i.e. informational status codes too).
8464 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8465 * have been disabled.
8466 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8467 * double-fault into the guest.
8468 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8469 * dispatched directly.
8470 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8471 *
8472 * @param pVCpu The cross context virtual CPU structure.
8473 * @param pVmxTransient Pointer to the VMX transient structure.
8474 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8475 * us ignore some of the reasons for returning to
8476 * ring-3, and return VINF_EM_DBG_STEPPED if event
8477 * dispatching took place.
8478 */
8479static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8480{
8481 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8482
8483#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8484 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8485 return VINF_EM_RESCHEDULE_REM;
8486#endif
8487
8488#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8489 PGMRZDynMapFlushAutoSet(pVCpu);
8490#endif
8491
8492 /* Check force flag actions that might require us to go back to ring-3. */
8493 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8494 if (rcStrict == VINF_SUCCESS)
8495 { /* FFs doesn't get set all the time. */ }
8496 else
8497 return rcStrict;
8498
8499 /*
8500 * Setup the virtualized-APIC accesses.
8501 *
8502 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8503 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8504 *
8505 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8506 */
8507 PVM pVM = pVCpu->CTX_SUFF(pVM);
8508 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8509 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8510 && PDMHasApic(pVM))
8511 {
8512 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8513 Assert(u64MsrApicBase);
8514 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8515
8516 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8517
8518 /* Unalias any existing mapping. */
8519 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8520 AssertRCReturn(rc, rc);
8521
8522 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8523 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8524 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8525 AssertRCReturn(rc, rc);
8526
8527 /* Update the per-VCPU cache of the APIC base MSR. */
8528 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8529 }
8530
8531 if (TRPMHasTrap(pVCpu))
8532 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8533 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8534
8535 /*
8536 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8537 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8538 * also result in triple-faulting the VM.
8539 */
8540 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8541 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8542 { /* likely */ }
8543 else
8544 {
8545 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8546 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8547 return rcStrict;
8548 }
8549
8550 /*
8551 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8552 * import CR3 themselves. We will need to update them here, as even as late as the above
8553 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8554 * the below force flags to be set.
8555 */
8556 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8557 {
8558 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8559 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8560 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8561 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8562 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8563 }
8564 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8565 {
8566 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8567 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8568 }
8569
8570 /*
8571 * No longjmps to ring-3 from this point on!!!
8572 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8573 * This also disables flushing of the R0-logger instance (if any).
8574 */
8575 VMMRZCallRing3Disable(pVCpu);
8576
8577 /*
8578 * Export the guest state bits.
8579 *
8580 * We cannot perform longjmps while loading the guest state because we do not preserve the
8581 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8582 * CPU migration.
8583 *
8584 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8585 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8586 * Hence, loading of the guest state needs to be done -after- injection of events.
8587 */
8588 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8589 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8590 { /* likely */ }
8591 else
8592 {
8593 VMMRZCallRing3Enable(pVCpu);
8594 return rcStrict;
8595 }
8596
8597 /*
8598 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8599 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8600 * preemption disabled for a while. Since this is purly to aid the
8601 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8602 * disable interrupt on NT.
8603 *
8604 * We need to check for force-flags that could've possible been altered since we last
8605 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8606 * see @bugref{6398}).
8607 *
8608 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8609 * to ring-3 before executing guest code.
8610 */
8611 pVmxTransient->fEFlags = ASMIntDisableFlags();
8612
8613 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8614 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8615 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8616 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8617 {
8618 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8619 {
8620 pVCpu->hm.s.Event.fPending = false;
8621
8622 /*
8623 * We've injected any pending events. This is really the point of no return (to ring-3).
8624 *
8625 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8626 * returns from this function, so don't enable them here.
8627 */
8628 return VINF_SUCCESS;
8629 }
8630
8631 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8632 rcStrict = VINF_EM_RAW_INTERRUPT;
8633 }
8634 else
8635 {
8636 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8637 rcStrict = VINF_EM_RAW_TO_R3;
8638 }
8639
8640 ASMSetFlags(pVmxTransient->fEFlags);
8641 VMMRZCallRing3Enable(pVCpu);
8642
8643 return rcStrict;
8644}
8645
8646
8647/**
8648 * Prepares to run guest code in VT-x and we've committed to doing so. This
8649 * means there is no backing out to ring-3 or anywhere else at this
8650 * point.
8651 *
8652 * @param pVCpu The cross context virtual CPU structure.
8653 * @param pVmxTransient Pointer to the VMX transient structure.
8654 *
8655 * @remarks Called with preemption disabled.
8656 * @remarks No-long-jump zone!!!
8657 */
8658static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8659{
8660 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8661 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8662 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8663
8664 /*
8665 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8666 */
8667 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8668 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8669
8670 PVM pVM = pVCpu->CTX_SUFF(pVM);
8671 if (!CPUMIsGuestFPUStateActive(pVCpu))
8672 {
8673 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8674 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8675 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8676 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8677 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8678 }
8679
8680 /*
8681 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8682 */
8683 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8684 && pVCpu->hm.s.vmx.cMsrs > 0)
8685 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8686
8687 /*
8688 * Re-save the host state bits as we may've been preempted (only happens when
8689 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8690 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8691 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8692 * See @bugref{8432}.
8693 */
8694 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8695 {
8696 int rc = hmR0VmxExportHostState(pVCpu);
8697 AssertRC(rc);
8698 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8699 }
8700 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8701
8702 /*
8703 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8704 */
8705 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8706 hmR0VmxExportSharedState(pVCpu);
8707 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8708
8709 /* Store status of the shared guest-host state at the time of VM-entry. */
8710#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8711 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8712 {
8713 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8714 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8715 }
8716 else
8717#endif
8718 {
8719 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8720 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8721 }
8722
8723 /*
8724 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8725 */
8726 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8727 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8728
8729 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8730 RTCPUID idCurrentCpu = pCpu->idCpu;
8731 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8732 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8733 {
8734 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8735 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8736 }
8737
8738 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8739 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8740 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8741 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8742
8743 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8744
8745 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8746 to start executing. */
8747
8748 /*
8749 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8750 */
8751 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8752 {
8753 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8754 {
8755 bool fMsrUpdated;
8756 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8757 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8758 &fMsrUpdated);
8759 AssertRC(rc2);
8760 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8761 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8762 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8763 }
8764 else
8765 {
8766 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8767 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8768 }
8769 }
8770
8771 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8772 {
8773 bool fMsrUpdated;
8774 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8775 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8776 &fMsrUpdated);
8777 AssertRC(rc2);
8778 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8779 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8780 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8781 }
8782
8783#ifdef VBOX_STRICT
8784 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8785 hmR0VmxCheckHostEferMsr(pVCpu);
8786 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8787#endif
8788#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8789 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8790 {
8791 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8792 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8793 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8794 }
8795#endif
8796}
8797
8798
8799/**
8800 * Performs some essential restoration of state after running guest code in
8801 * VT-x.
8802 *
8803 * @param pVCpu The cross context virtual CPU structure.
8804 * @param pVmxTransient Pointer to the VMX transient structure.
8805 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8806 *
8807 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8808 *
8809 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8810 * unconditionally when it is safe to do so.
8811 */
8812static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8813{
8814 uint64_t const uHostTsc = ASMReadTSC();
8815 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8816
8817 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8818 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8819 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8820 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8821 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8822 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8823
8824 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8825 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8826
8827 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8828 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8829 Assert(!ASMIntAreEnabled());
8830 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8831
8832#if HC_ARCH_BITS == 64
8833 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8834#endif
8835#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8836 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8837 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8838 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8839#else
8840 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8841#endif
8842#ifdef VBOX_STRICT
8843 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8844#endif
8845 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8846
8847 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8848 uint32_t uExitReason;
8849 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8850 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8851 AssertRC(rc);
8852 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8853 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INT_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8854
8855 if (rcVMRun == VINF_SUCCESS)
8856 {
8857 /*
8858 * Update the VM-exit history array here even if the VM-entry failed due to:
8859 * - Invalid guest state.
8860 * - MSR loading.
8861 * - Machine-check event.
8862 *
8863 * In any of the above cases we will still have a "valid" VM-exit reason
8864 * despite @a fVMEntryFailed being false.
8865 *
8866 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8867 *
8868 * Note! We don't have CS or RIP at this point. Will probably address that later
8869 * by amending the history entry added here.
8870 */
8871 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8872 UINT64_MAX, uHostTsc);
8873
8874 if (!pVmxTransient->fVMEntryFailed)
8875 {
8876 VMMRZCallRing3Enable(pVCpu);
8877
8878 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8879 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8880
8881#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8882 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8883 AssertRC(rc);
8884#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8885 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8886 AssertRC(rc);
8887#else
8888 /*
8889 * Import the guest-interruptibility state always as we need it while evaluating
8890 * injecting events on re-entry.
8891 *
8892 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8893 * checking for real-mode while exporting the state because all bits that cause
8894 * mode changes wrt CR0 are intercepted.
8895 */
8896 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8897 AssertRC(rc);
8898#endif
8899
8900 /*
8901 * Sync the TPR shadow with our APIC state.
8902 */
8903 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8904 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8905 {
8906 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8907 AssertRC(rc);
8908 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8909 }
8910
8911 return;
8912 }
8913 }
8914 else
8915 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8916
8917 VMMRZCallRing3Enable(pVCpu);
8918}
8919
8920
8921/**
8922 * Runs the guest code using VT-x the normal way.
8923 *
8924 * @returns VBox status code.
8925 * @param pVCpu The cross context virtual CPU structure.
8926 *
8927 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8928 */
8929static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8930{
8931 VMXTRANSIENT VmxTransient;
8932 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8933 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8934 uint32_t cLoops = 0;
8935
8936 for (;; cLoops++)
8937 {
8938 Assert(!HMR0SuspendPending());
8939 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8940
8941 /* Preparatory work for running guest code, this may force us to return
8942 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8943 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8944 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8945 if (rcStrict != VINF_SUCCESS)
8946 break;
8947
8948 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8949 int rcRun = hmR0VmxRunGuest(pVCpu);
8950
8951 /* Restore any residual host-state and save any bits shared between host
8952 and guest into the guest-CPU state. Re-enables interrupts! */
8953 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8954
8955 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8956 if (RT_SUCCESS(rcRun))
8957 { /* very likely */ }
8958 else
8959 {
8960 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8961 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8962 return rcRun;
8963 }
8964
8965 /* Profile the VM-exit. */
8966 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8967 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8968 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8969 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8970 HMVMX_START_EXIT_DISPATCH_PROF();
8971
8972 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8973
8974 /* Handle the VM-exit. */
8975#ifdef HMVMX_USE_FUNCTION_TABLE
8976 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8977#else
8978 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8979#endif
8980 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8981 if (rcStrict == VINF_SUCCESS)
8982 {
8983 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8984 continue; /* likely */
8985 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8986 rcStrict = VINF_EM_RAW_INTERRUPT;
8987 }
8988 break;
8989 }
8990
8991 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8992 return rcStrict;
8993}
8994
8995
8996
8997/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8998 * probes.
8999 *
9000 * The following few functions and associated structure contains the bloat
9001 * necessary for providing detailed debug events and dtrace probes as well as
9002 * reliable host side single stepping. This works on the principle of
9003 * "subclassing" the normal execution loop and workers. We replace the loop
9004 * method completely and override selected helpers to add necessary adjustments
9005 * to their core operation.
9006 *
9007 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9008 * any performance for debug and analysis features.
9009 *
9010 * @{
9011 */
9012
9013/**
9014 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9015 * the debug run loop.
9016 */
9017typedef struct VMXRUNDBGSTATE
9018{
9019 /** The RIP we started executing at. This is for detecting that we stepped. */
9020 uint64_t uRipStart;
9021 /** The CS we started executing with. */
9022 uint16_t uCsStart;
9023
9024 /** Whether we've actually modified the 1st execution control field. */
9025 bool fModifiedProcCtls : 1;
9026 /** Whether we've actually modified the 2nd execution control field. */
9027 bool fModifiedProcCtls2 : 1;
9028 /** Whether we've actually modified the exception bitmap. */
9029 bool fModifiedXcptBitmap : 1;
9030
9031 /** We desire the modified the CR0 mask to be cleared. */
9032 bool fClearCr0Mask : 1;
9033 /** We desire the modified the CR4 mask to be cleared. */
9034 bool fClearCr4Mask : 1;
9035 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9036 uint32_t fCpe1Extra;
9037 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9038 uint32_t fCpe1Unwanted;
9039 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9040 uint32_t fCpe2Extra;
9041 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9042 uint32_t bmXcptExtra;
9043 /** The sequence number of the Dtrace provider settings the state was
9044 * configured against. */
9045 uint32_t uDtraceSettingsSeqNo;
9046 /** VM-exits to check (one bit per VM-exit). */
9047 uint32_t bmExitsToCheck[3];
9048
9049 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9050 uint32_t fProcCtlsInitial;
9051 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9052 uint32_t fProcCtls2Initial;
9053 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9054 uint32_t bmXcptInitial;
9055} VMXRUNDBGSTATE;
9056AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9057typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9058
9059
9060/**
9061 * Initializes the VMXRUNDBGSTATE structure.
9062 *
9063 * @param pVCpu The cross context virtual CPU structure of the
9064 * calling EMT.
9065 * @param pDbgState The structure to initialize.
9066 */
9067static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9068{
9069 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9070 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9071
9072 pDbgState->fModifiedProcCtls = false;
9073 pDbgState->fModifiedProcCtls2 = false;
9074 pDbgState->fModifiedXcptBitmap = false;
9075 pDbgState->fClearCr0Mask = false;
9076 pDbgState->fClearCr4Mask = false;
9077 pDbgState->fCpe1Extra = 0;
9078 pDbgState->fCpe1Unwanted = 0;
9079 pDbgState->fCpe2Extra = 0;
9080 pDbgState->bmXcptExtra = 0;
9081 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9082 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9083 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9084}
9085
9086
9087/**
9088 * Updates the VMSC fields with changes requested by @a pDbgState.
9089 *
9090 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9091 * immediately before executing guest code, i.e. when interrupts are disabled.
9092 * We don't check status codes here as we cannot easily assert or return in the
9093 * latter case.
9094 *
9095 * @param pVCpu The cross context virtual CPU structure.
9096 * @param pDbgState The debug state.
9097 */
9098static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9099{
9100 /*
9101 * Ensure desired flags in VMCS control fields are set.
9102 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9103 *
9104 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9105 * there should be no stale data in pCtx at this point.
9106 */
9107 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9108 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9109 {
9110 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9111 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9112 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9113 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9114 pDbgState->fModifiedProcCtls = true;
9115 }
9116
9117 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9118 {
9119 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9120 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9121 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9122 pDbgState->fModifiedProcCtls2 = true;
9123 }
9124
9125 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9126 {
9127 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9128 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9129 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9130 pDbgState->fModifiedXcptBitmap = true;
9131 }
9132
9133 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9134 {
9135 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9136 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9137 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9138 }
9139
9140 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9141 {
9142 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9143 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9144 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9145 }
9146}
9147
9148
9149/**
9150 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9151 * re-entry next time around.
9152 *
9153 * @returns Strict VBox status code (i.e. informational status codes too).
9154 * @param pVCpu The cross context virtual CPU structure.
9155 * @param pDbgState The debug state.
9156 * @param rcStrict The return code from executing the guest using single
9157 * stepping.
9158 */
9159static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9160{
9161 /*
9162 * Restore VM-exit control settings as we may not reenter this function the
9163 * next time around.
9164 */
9165 /* We reload the initial value, trigger what we can of recalculations the
9166 next time around. From the looks of things, that's all that's required atm. */
9167 if (pDbgState->fModifiedProcCtls)
9168 {
9169 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9170 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9171 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9172 AssertRCReturn(rc2, rc2);
9173 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9174 }
9175
9176 /* We're currently the only ones messing with this one, so just restore the
9177 cached value and reload the field. */
9178 if ( pDbgState->fModifiedProcCtls2
9179 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9180 {
9181 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9182 AssertRCReturn(rc2, rc2);
9183 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9184 }
9185
9186 /* If we've modified the exception bitmap, we restore it and trigger
9187 reloading and partial recalculation the next time around. */
9188 if (pDbgState->fModifiedXcptBitmap)
9189 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9190
9191 return rcStrict;
9192}
9193
9194
9195/**
9196 * Configures VM-exit controls for current DBGF and DTrace settings.
9197 *
9198 * This updates @a pDbgState and the VMCS execution control fields to reflect
9199 * the necessary VM-exits demanded by DBGF and DTrace.
9200 *
9201 * @param pVCpu The cross context virtual CPU structure.
9202 * @param pDbgState The debug state.
9203 * @param pVmxTransient Pointer to the VMX transient structure. May update
9204 * fUpdateTscOffsettingAndPreemptTimer.
9205 */
9206static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9207{
9208 /*
9209 * Take down the dtrace serial number so we can spot changes.
9210 */
9211 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9212 ASMCompilerBarrier();
9213
9214 /*
9215 * We'll rebuild most of the middle block of data members (holding the
9216 * current settings) as we go along here, so start by clearing it all.
9217 */
9218 pDbgState->bmXcptExtra = 0;
9219 pDbgState->fCpe1Extra = 0;
9220 pDbgState->fCpe1Unwanted = 0;
9221 pDbgState->fCpe2Extra = 0;
9222 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9223 pDbgState->bmExitsToCheck[i] = 0;
9224
9225 /*
9226 * Software interrupts (INT XXh) - no idea how to trigger these...
9227 */
9228 PVM pVM = pVCpu->CTX_SUFF(pVM);
9229 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9230 || VBOXVMM_INT_SOFTWARE_ENABLED())
9231 {
9232 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9233 }
9234
9235 /*
9236 * INT3 breakpoints - triggered by #BP exceptions.
9237 */
9238 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9239 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9240
9241 /*
9242 * Exception bitmap and XCPT events+probes.
9243 */
9244 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9245 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9246 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9247
9248 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9249 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9250 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9251 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9252 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9253 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9254 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9255 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9256 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9257 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9258 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9259 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9260 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9261 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9262 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9263 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9264 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9265 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9266
9267 if (pDbgState->bmXcptExtra)
9268 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9269
9270 /*
9271 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9272 *
9273 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9274 * So, when adding/changing/removing please don't forget to update it.
9275 *
9276 * Some of the macros are picking up local variables to save horizontal space,
9277 * (being able to see it in a table is the lesser evil here).
9278 */
9279#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9280 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9281 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9282#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9283 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9284 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9285 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9286 } else do { } while (0)
9287#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9288 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9289 { \
9290 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9291 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9292 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9293 } else do { } while (0)
9294#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9295 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9296 { \
9297 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9298 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9299 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9300 } else do { } while (0)
9301#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9302 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9303 { \
9304 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9305 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9306 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9307 } else do { } while (0)
9308
9309 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9310 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9311 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9312 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9313 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9314
9315 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9316 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9317 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9318 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9319 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9320 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9321 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9322 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9323 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9324 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9325 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9326 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9327 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9328 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9329 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9330 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9331 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9332 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9333 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9334 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9335 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9336 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9337 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9338 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9339 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9340 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9341 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9342 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9343 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9344 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9345 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9346 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9347 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9348 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9349 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9350 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9351
9352 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9353 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9354 {
9355 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9356 AssertRC(rc);
9357
9358#if 0 /** @todo fix me */
9359 pDbgState->fClearCr0Mask = true;
9360 pDbgState->fClearCr4Mask = true;
9361#endif
9362 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9363 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9364 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9365 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9366 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9367 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9368 require clearing here and in the loop if we start using it. */
9369 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9370 }
9371 else
9372 {
9373 if (pDbgState->fClearCr0Mask)
9374 {
9375 pDbgState->fClearCr0Mask = false;
9376 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9377 }
9378 if (pDbgState->fClearCr4Mask)
9379 {
9380 pDbgState->fClearCr4Mask = false;
9381 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9382 }
9383 }
9384 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9385 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9386
9387 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9388 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9389 {
9390 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9391 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9392 }
9393 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9394 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9395
9396 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9397 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9398 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9399 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9400 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9402 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9403 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9404#if 0 /** @todo too slow, fix handler. */
9405 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9406#endif
9407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9408
9409 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9410 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9411 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9412 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9413 {
9414 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9415 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9416 }
9417 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9419 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9420 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9421
9422 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9423 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9424 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9425 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9426 {
9427 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9428 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9429 }
9430 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9431 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9432 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9433 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9434
9435 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9436 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9437 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9438 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9439 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9440 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9441 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9442 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9443 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9444 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9445 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9446 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9447 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9448 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9449 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9450 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9451 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9452 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9453 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9454 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9455 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9456 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9457
9458#undef IS_EITHER_ENABLED
9459#undef SET_ONLY_XBM_IF_EITHER_EN
9460#undef SET_CPE1_XBM_IF_EITHER_EN
9461#undef SET_CPEU_XBM_IF_EITHER_EN
9462#undef SET_CPE2_XBM_IF_EITHER_EN
9463
9464 /*
9465 * Sanitize the control stuff.
9466 */
9467 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9468 if (pDbgState->fCpe2Extra)
9469 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9470 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9471 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9472 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9473 {
9474 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9475 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9476 }
9477
9478 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9479 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9480 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9481 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9482}
9483
9484
9485/**
9486 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9487 * appropriate.
9488 *
9489 * The caller has checked the VM-exit against the
9490 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9491 * already, so we don't have to do that either.
9492 *
9493 * @returns Strict VBox status code (i.e. informational status codes too).
9494 * @param pVCpu The cross context virtual CPU structure.
9495 * @param pVmxTransient Pointer to the VMX-transient structure.
9496 * @param uExitReason The VM-exit reason.
9497 *
9498 * @remarks The name of this function is displayed by dtrace, so keep it short
9499 * and to the point. No longer than 33 chars long, please.
9500 */
9501static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9502{
9503 /*
9504 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9505 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9506 *
9507 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9508 * does. Must add/change/remove both places. Same ordering, please.
9509 *
9510 * Added/removed events must also be reflected in the next section
9511 * where we dispatch dtrace events.
9512 */
9513 bool fDtrace1 = false;
9514 bool fDtrace2 = false;
9515 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9516 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9517 uint32_t uEventArg = 0;
9518#define SET_EXIT(a_EventSubName) \
9519 do { \
9520 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9521 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9522 } while (0)
9523#define SET_BOTH(a_EventSubName) \
9524 do { \
9525 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9526 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9527 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9528 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9529 } while (0)
9530 switch (uExitReason)
9531 {
9532 case VMX_EXIT_MTF:
9533 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9534
9535 case VMX_EXIT_XCPT_OR_NMI:
9536 {
9537 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9538 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9539 {
9540 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9541 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9542 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9543 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9544 {
9545 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9546 {
9547 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9548 uEventArg = pVmxTransient->uExitIntErrorCode;
9549 }
9550 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9551 switch (enmEvent1)
9552 {
9553 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9554 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9555 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9556 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9557 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9558 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9559 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9560 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9561 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9562 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9563 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9564 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9565 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9566 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9567 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9568 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9569 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9570 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9571 default: break;
9572 }
9573 }
9574 else
9575 AssertFailed();
9576 break;
9577
9578 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9579 uEventArg = idxVector;
9580 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9581 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9582 break;
9583 }
9584 break;
9585 }
9586
9587 case VMX_EXIT_TRIPLE_FAULT:
9588 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9589 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9590 break;
9591 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9592 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9593 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9594 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9595 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9596
9597 /* Instruction specific VM-exits: */
9598 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9599 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9600 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9601 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9602 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9603 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9604 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9605 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9606 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9607 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9608 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9609 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9610 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9611 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9612 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9613 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9614 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9615 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9616 case VMX_EXIT_MOV_CRX:
9617 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9618 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9619 SET_BOTH(CRX_READ);
9620 else
9621 SET_BOTH(CRX_WRITE);
9622 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9623 break;
9624 case VMX_EXIT_MOV_DRX:
9625 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9626 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9627 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9628 SET_BOTH(DRX_READ);
9629 else
9630 SET_BOTH(DRX_WRITE);
9631 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9632 break;
9633 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9634 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9635 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9636 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9637 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9638 case VMX_EXIT_XDTR_ACCESS:
9639 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9640 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9641 {
9642 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9643 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9644 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9645 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9646 }
9647 break;
9648
9649 case VMX_EXIT_TR_ACCESS:
9650 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9651 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9652 {
9653 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9654 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9655 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9656 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9657 }
9658 break;
9659
9660 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9661 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9662 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9663 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9664 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9665 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9666 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9667 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9668 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9669 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9670 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9671
9672 /* Events that aren't relevant at this point. */
9673 case VMX_EXIT_EXT_INT:
9674 case VMX_EXIT_INT_WINDOW:
9675 case VMX_EXIT_NMI_WINDOW:
9676 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9677 case VMX_EXIT_PREEMPT_TIMER:
9678 case VMX_EXIT_IO_INSTR:
9679 break;
9680
9681 /* Errors and unexpected events. */
9682 case VMX_EXIT_INIT_SIGNAL:
9683 case VMX_EXIT_SIPI:
9684 case VMX_EXIT_IO_SMI:
9685 case VMX_EXIT_SMI:
9686 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9687 case VMX_EXIT_ERR_MSR_LOAD:
9688 case VMX_EXIT_ERR_MACHINE_CHECK:
9689 break;
9690
9691 default:
9692 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9693 break;
9694 }
9695#undef SET_BOTH
9696#undef SET_EXIT
9697
9698 /*
9699 * Dtrace tracepoints go first. We do them here at once so we don't
9700 * have to copy the guest state saving and stuff a few dozen times.
9701 * Down side is that we've got to repeat the switch, though this time
9702 * we use enmEvent since the probes are a subset of what DBGF does.
9703 */
9704 if (fDtrace1 || fDtrace2)
9705 {
9706 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9707 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9708 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9709 switch (enmEvent1)
9710 {
9711 /** @todo consider which extra parameters would be helpful for each probe. */
9712 case DBGFEVENT_END: break;
9713 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9714 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9715 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9716 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9717 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9718 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9719 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9720 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9721 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9722 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9723 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9724 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9725 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9726 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9727 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9728 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9729 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9730 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9731 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9732 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9733 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9734 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9735 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9736 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9737 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9738 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9739 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9740 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9741 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9742 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9743 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9744 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9745 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9746 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9747 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9748 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9749 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9750 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9751 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9752 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9753 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9754 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9755 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9756 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9757 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9758 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9759 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9760 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9761 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9762 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9763 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9764 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9765 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9766 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9767 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9768 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9769 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9770 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9771 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9772 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9773 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9774 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9775 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9776 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9777 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9778 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9779 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9780 }
9781 switch (enmEvent2)
9782 {
9783 /** @todo consider which extra parameters would be helpful for each probe. */
9784 case DBGFEVENT_END: break;
9785 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9786 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9787 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9788 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9789 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9790 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9791 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9792 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9793 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9794 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9795 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9796 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9797 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9798 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9799 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9800 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9801 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9802 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9803 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9804 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9805 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9806 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9807 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9808 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9809 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9810 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9811 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9812 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9813 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9814 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9815 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9816 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9817 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9818 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9819 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9820 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9821 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9822 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9823 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9824 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9825 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9826 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9827 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9828 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9829 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9830 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9831 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9832 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9833 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9834 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9835 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9836 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9837 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9838 }
9839 }
9840
9841 /*
9842 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9843 * the DBGF call will do a full check).
9844 *
9845 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9846 * Note! If we have to events, we prioritize the first, i.e. the instruction
9847 * one, in order to avoid event nesting.
9848 */
9849 PVM pVM = pVCpu->CTX_SUFF(pVM);
9850 if ( enmEvent1 != DBGFEVENT_END
9851 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9852 {
9853 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9854 if (rcStrict != VINF_SUCCESS)
9855 return rcStrict;
9856 }
9857 else if ( enmEvent2 != DBGFEVENT_END
9858 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9859 {
9860 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9861 if (rcStrict != VINF_SUCCESS)
9862 return rcStrict;
9863 }
9864
9865 return VINF_SUCCESS;
9866}
9867
9868
9869/**
9870 * Single-stepping VM-exit filtering.
9871 *
9872 * This is preprocessing the VM-exits and deciding whether we've gotten far
9873 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9874 * handling is performed.
9875 *
9876 * @returns Strict VBox status code (i.e. informational status codes too).
9877 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9878 * @param pVmxTransient Pointer to the VMX-transient structure.
9879 * @param pDbgState The debug state.
9880 */
9881DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9882{
9883 /*
9884 * Expensive (saves context) generic dtrace VM-exit probe.
9885 */
9886 uint32_t const uExitReason = pVmxTransient->uExitReason;
9887 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9888 { /* more likely */ }
9889 else
9890 {
9891 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9892 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9893 AssertRC(rc);
9894 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9895 }
9896
9897 /*
9898 * Check for host NMI, just to get that out of the way.
9899 */
9900 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9901 { /* normally likely */ }
9902 else
9903 {
9904 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9905 AssertRCReturn(rc2, rc2);
9906 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9907 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9908 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9909 }
9910
9911 /*
9912 * Check for single stepping event if we're stepping.
9913 */
9914 if (pVCpu->hm.s.fSingleInstruction)
9915 {
9916 switch (uExitReason)
9917 {
9918 case VMX_EXIT_MTF:
9919 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9920
9921 /* Various events: */
9922 case VMX_EXIT_XCPT_OR_NMI:
9923 case VMX_EXIT_EXT_INT:
9924 case VMX_EXIT_TRIPLE_FAULT:
9925 case VMX_EXIT_INT_WINDOW:
9926 case VMX_EXIT_NMI_WINDOW:
9927 case VMX_EXIT_TASK_SWITCH:
9928 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9929 case VMX_EXIT_APIC_ACCESS:
9930 case VMX_EXIT_EPT_VIOLATION:
9931 case VMX_EXIT_EPT_MISCONFIG:
9932 case VMX_EXIT_PREEMPT_TIMER:
9933
9934 /* Instruction specific VM-exits: */
9935 case VMX_EXIT_CPUID:
9936 case VMX_EXIT_GETSEC:
9937 case VMX_EXIT_HLT:
9938 case VMX_EXIT_INVD:
9939 case VMX_EXIT_INVLPG:
9940 case VMX_EXIT_RDPMC:
9941 case VMX_EXIT_RDTSC:
9942 case VMX_EXIT_RSM:
9943 case VMX_EXIT_VMCALL:
9944 case VMX_EXIT_VMCLEAR:
9945 case VMX_EXIT_VMLAUNCH:
9946 case VMX_EXIT_VMPTRLD:
9947 case VMX_EXIT_VMPTRST:
9948 case VMX_EXIT_VMREAD:
9949 case VMX_EXIT_VMRESUME:
9950 case VMX_EXIT_VMWRITE:
9951 case VMX_EXIT_VMXOFF:
9952 case VMX_EXIT_VMXON:
9953 case VMX_EXIT_MOV_CRX:
9954 case VMX_EXIT_MOV_DRX:
9955 case VMX_EXIT_IO_INSTR:
9956 case VMX_EXIT_RDMSR:
9957 case VMX_EXIT_WRMSR:
9958 case VMX_EXIT_MWAIT:
9959 case VMX_EXIT_MONITOR:
9960 case VMX_EXIT_PAUSE:
9961 case VMX_EXIT_XDTR_ACCESS:
9962 case VMX_EXIT_TR_ACCESS:
9963 case VMX_EXIT_INVEPT:
9964 case VMX_EXIT_RDTSCP:
9965 case VMX_EXIT_INVVPID:
9966 case VMX_EXIT_WBINVD:
9967 case VMX_EXIT_XSETBV:
9968 case VMX_EXIT_RDRAND:
9969 case VMX_EXIT_INVPCID:
9970 case VMX_EXIT_VMFUNC:
9971 case VMX_EXIT_RDSEED:
9972 case VMX_EXIT_XSAVES:
9973 case VMX_EXIT_XRSTORS:
9974 {
9975 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9976 AssertRCReturn(rc, rc);
9977 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9978 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9979 return VINF_EM_DBG_STEPPED;
9980 break;
9981 }
9982
9983 /* Errors and unexpected events: */
9984 case VMX_EXIT_INIT_SIGNAL:
9985 case VMX_EXIT_SIPI:
9986 case VMX_EXIT_IO_SMI:
9987 case VMX_EXIT_SMI:
9988 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9989 case VMX_EXIT_ERR_MSR_LOAD:
9990 case VMX_EXIT_ERR_MACHINE_CHECK:
9991 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9992 break;
9993
9994 default:
9995 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9996 break;
9997 }
9998 }
9999
10000 /*
10001 * Check for debugger event breakpoints and dtrace probes.
10002 */
10003 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10004 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10005 {
10006 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
10007 if (rcStrict != VINF_SUCCESS)
10008 return rcStrict;
10009 }
10010
10011 /*
10012 * Normal processing.
10013 */
10014#ifdef HMVMX_USE_FUNCTION_TABLE
10015 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
10016#else
10017 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
10018#endif
10019}
10020
10021
10022/**
10023 * Single steps guest code using VT-x.
10024 *
10025 * @returns Strict VBox status code (i.e. informational status codes too).
10026 * @param pVCpu The cross context virtual CPU structure.
10027 *
10028 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10029 */
10030static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
10031{
10032 VMXTRANSIENT VmxTransient;
10033 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10034
10035 /* Set HMCPU indicators. */
10036 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10037 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10038 pVCpu->hm.s.fDebugWantRdTscExit = false;
10039 pVCpu->hm.s.fUsingDebugLoop = true;
10040
10041 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10042 VMXRUNDBGSTATE DbgState;
10043 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
10044 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10045
10046 /*
10047 * The loop.
10048 */
10049 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10050 for (uint32_t cLoops = 0; ; cLoops++)
10051 {
10052 Assert(!HMR0SuspendPending());
10053 HMVMX_ASSERT_CPU_SAFE(pVCpu);
10054 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10055
10056 /*
10057 * Preparatory work for running guest code, this may force us to return
10058 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10059 */
10060 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10061 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10062 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10063 if (rcStrict != VINF_SUCCESS)
10064 break;
10065
10066 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10067 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10068
10069 /*
10070 * Now we can run the guest code.
10071 */
10072 int rcRun = hmR0VmxRunGuest(pVCpu);
10073
10074 /*
10075 * Restore any residual host-state and save any bits shared between host
10076 * and guest into the guest-CPU state. Re-enables interrupts!
10077 */
10078 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10079
10080 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10081 if (RT_SUCCESS(rcRun))
10082 { /* very likely */ }
10083 else
10084 {
10085 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10086 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10087 return rcRun;
10088 }
10089
10090 /* Profile the VM-exit. */
10091 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10092 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10093 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10094 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10095 HMVMX_START_EXIT_DISPATCH_PROF();
10096
10097 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10098
10099 /*
10100 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10101 */
10102 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10103 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10104 if (rcStrict != VINF_SUCCESS)
10105 break;
10106 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10107 {
10108 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10109 rcStrict = VINF_EM_RAW_INTERRUPT;
10110 break;
10111 }
10112
10113 /*
10114 * Stepping: Did the RIP change, if so, consider it a single step.
10115 * Otherwise, make sure one of the TFs gets set.
10116 */
10117 if (fStepping)
10118 {
10119 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
10120 AssertRC(rc);
10121 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10122 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10123 {
10124 rcStrict = VINF_EM_DBG_STEPPED;
10125 break;
10126 }
10127 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10128 }
10129
10130 /*
10131 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10132 */
10133 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10134 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10135 }
10136
10137 /*
10138 * Clear the X86_EFL_TF if necessary.
10139 */
10140 if (pVCpu->hm.s.fClearTrapFlag)
10141 {
10142 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10143 AssertRC(rc);
10144 pVCpu->hm.s.fClearTrapFlag = false;
10145 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10146 }
10147 /** @todo there seems to be issues with the resume flag when the monitor trap
10148 * flag is pending without being used. Seen early in bios init when
10149 * accessing APIC page in protected mode. */
10150
10151 /*
10152 * Restore VM-exit control settings as we may not reenter this function the
10153 * next time around.
10154 */
10155 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10156
10157 /* Restore HMCPU indicators. */
10158 pVCpu->hm.s.fUsingDebugLoop = false;
10159 pVCpu->hm.s.fDebugWantRdTscExit = false;
10160 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10161
10162 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10163 return rcStrict;
10164}
10165
10166
10167/** @} */
10168
10169
10170/**
10171 * Checks if any expensive dtrace probes are enabled and we should go to the
10172 * debug loop.
10173 *
10174 * @returns true if we should use debug loop, false if not.
10175 */
10176static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10177{
10178 /* It's probably faster to OR the raw 32-bit counter variables together.
10179 Since the variables are in an array and the probes are next to one
10180 another (more or less), we have good locality. So, better read
10181 eight-nine cache lines ever time and only have one conditional, than
10182 128+ conditionals, right? */
10183 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10184 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10185 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10186 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10187 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10188 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10189 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10190 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10191 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10192 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10193 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10194 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10195 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10196 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10197 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10198 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10199 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10200 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10201 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10202 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10203 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10204 ) != 0
10205 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10206 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10207 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10208 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10209 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10210 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10211 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10212 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10213 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10214 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10215 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10216 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10217 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10218 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10219 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10220 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10221 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10222 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10223 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10224 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10225 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10226 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10227 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10228 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10229 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10230 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10231 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10232 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10233 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10234 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10235 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10236 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10237 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10238 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10239 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10240 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10241 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10242 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10243 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10244 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10245 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10246 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10247 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10248 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10249 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10250 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10251 ) != 0
10252 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10253 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10254 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10255 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10256 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10257 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10258 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10259 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10260 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10261 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10262 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10263 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10264 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10265 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10266 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10267 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10268 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10269 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10270 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10271 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10272 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10273 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10274 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10275 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10276 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10277 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10278 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10279 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10280 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10281 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10282 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10283 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10284 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10285 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10286 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10287 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10288 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10289 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10290 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10291 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10292 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10293 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10294 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10295 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10296 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10297 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10298 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10299 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10300 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10301 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10302 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10303 ) != 0;
10304}
10305
10306
10307/**
10308 * Runs the guest code using VT-x.
10309 *
10310 * @returns Strict VBox status code (i.e. informational status codes too).
10311 * @param pVCpu The cross context virtual CPU structure.
10312 */
10313VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10314{
10315 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10316 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10317 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10318 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10319
10320 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10321
10322 VBOXSTRICTRC rcStrict;
10323 if ( !pVCpu->hm.s.fUseDebugLoop
10324 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10325 && !DBGFIsStepping(pVCpu)
10326 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10327 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10328 else
10329 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10330
10331 if (rcStrict == VERR_EM_INTERPRETER)
10332 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10333 else if (rcStrict == VINF_EM_RESET)
10334 rcStrict = VINF_EM_TRIPLE_FAULT;
10335
10336 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10337 if (RT_FAILURE(rc2))
10338 {
10339 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10340 rcStrict = rc2;
10341 }
10342 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10343 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10344 return rcStrict;
10345}
10346
10347
10348#ifndef HMVMX_USE_FUNCTION_TABLE
10349DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10350{
10351#ifdef DEBUG_ramshankar
10352#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10353 do { \
10354 if (a_fSave != 0) \
10355 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10356 VBOXSTRICTRC rcStrict = a_CallExpr; \
10357 if (a_fSave != 0) \
10358 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10359 return rcStrict; \
10360 } while (0)
10361#else
10362# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10363#endif
10364 switch (rcReason)
10365 {
10366 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10367 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10368 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10369 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10370 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10371 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10372 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10373 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10374 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10375 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10376 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10377 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10378 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10379 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10380 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10381 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10382 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10383 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10384 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10385 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10386 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10387 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10388 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10389 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10390 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10391 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10392 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10393 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10394 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10395 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10396 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10397 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10398 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10399 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10400
10401 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10402 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10403 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10404 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10405 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10406 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10407 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10408 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10409 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10410
10411 case VMX_EXIT_VMCLEAR:
10412 case VMX_EXIT_VMLAUNCH:
10413 case VMX_EXIT_VMPTRLD:
10414 case VMX_EXIT_VMPTRST:
10415 case VMX_EXIT_VMREAD:
10416 case VMX_EXIT_VMRESUME:
10417 case VMX_EXIT_VMWRITE:
10418 case VMX_EXIT_VMXOFF:
10419 case VMX_EXIT_VMXON:
10420 case VMX_EXIT_INVEPT:
10421 case VMX_EXIT_INVVPID:
10422 case VMX_EXIT_VMFUNC:
10423 case VMX_EXIT_XSAVES:
10424 case VMX_EXIT_XRSTORS:
10425 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10426
10427 case VMX_EXIT_ENCLS:
10428 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10429 case VMX_EXIT_PML_FULL:
10430 default:
10431 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10432 }
10433#undef VMEXIT_CALL_RET
10434}
10435#endif /* !HMVMX_USE_FUNCTION_TABLE */
10436
10437
10438#ifdef VBOX_STRICT
10439/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10440# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10441 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10442
10443# define HMVMX_ASSERT_PREEMPT_CPUID() \
10444 do { \
10445 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10446 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10447 } while (0)
10448
10449# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10450 do { \
10451 AssertPtr((a_pVCpu)); \
10452 AssertPtr((a_pVmxTransient)); \
10453 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10454 Assert(ASMIntAreEnabled()); \
10455 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10456 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10457 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10458 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10459 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10460 HMVMX_ASSERT_PREEMPT_CPUID(); \
10461 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10462 } while (0)
10463
10464# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10465 do { \
10466 Log4Func(("\n")); \
10467 } while (0)
10468#else
10469# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10470 do { \
10471 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10472 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10473 } while (0)
10474# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10475#endif
10476
10477
10478/**
10479 * Advances the guest RIP by the specified number of bytes.
10480 *
10481 * @param pVCpu The cross context virtual CPU structure.
10482 * @param cbInstr Number of bytes to advance the RIP by.
10483 *
10484 * @remarks No-long-jump zone!!!
10485 */
10486DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10487{
10488 /* Advance the RIP. */
10489 pVCpu->cpum.GstCtx.rip += cbInstr;
10490 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10491
10492 /* Update interrupt inhibition. */
10493 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10494 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10495 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10496}
10497
10498
10499/**
10500 * Advances the guest RIP after reading it from the VMCS.
10501 *
10502 * @returns VBox status code, no informational status codes.
10503 * @param pVCpu The cross context virtual CPU structure.
10504 * @param pVmxTransient Pointer to the VMX transient structure.
10505 *
10506 * @remarks No-long-jump zone!!!
10507 */
10508static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10509{
10510 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10511 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10512 AssertRCReturn(rc, rc);
10513
10514 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10515
10516 /*
10517 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10518 * pending debug exception field as it takes care of priority of events.
10519 *
10520 * See Intel spec. 32.2.1 "Debug Exceptions".
10521 */
10522 if ( !pVCpu->hm.s.fSingleInstruction
10523 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10524 {
10525 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10526 AssertRCReturn(rc, rc);
10527 }
10528
10529 return VINF_SUCCESS;
10530}
10531
10532
10533/**
10534 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10535 * and update error record fields accordingly.
10536 *
10537 * @return VMX_IGS_* return codes.
10538 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10539 * wrong with the guest state.
10540 *
10541 * @param pVCpu The cross context virtual CPU structure.
10542 *
10543 * @remarks This function assumes our cache of the VMCS controls
10544 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10545 */
10546static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10547{
10548#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10549#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10550 uError = (err); \
10551 break; \
10552 } else do { } while (0)
10553
10554 int rc;
10555 PVM pVM = pVCpu->CTX_SUFF(pVM);
10556 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10557 uint32_t uError = VMX_IGS_ERROR;
10558 uint32_t u32Val;
10559 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10560
10561 do
10562 {
10563 /*
10564 * CR0.
10565 */
10566 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10567 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10568 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10569 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10570 if (fUnrestrictedGuest)
10571 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10572
10573 uint32_t u32GuestCr0;
10574 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10575 AssertRCBreak(rc);
10576 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10577 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10578 if ( !fUnrestrictedGuest
10579 && (u32GuestCr0 & X86_CR0_PG)
10580 && !(u32GuestCr0 & X86_CR0_PE))
10581 {
10582 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10583 }
10584
10585 /*
10586 * CR4.
10587 */
10588 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10589 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10590
10591 uint32_t u32GuestCr4;
10592 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10593 AssertRCBreak(rc);
10594 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10595 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10596
10597 /*
10598 * IA32_DEBUGCTL MSR.
10599 */
10600 uint64_t u64Val;
10601 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10602 AssertRCBreak(rc);
10603 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10604 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10605 {
10606 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10607 }
10608 uint64_t u64DebugCtlMsr = u64Val;
10609
10610#ifdef VBOX_STRICT
10611 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10612 AssertRCBreak(rc);
10613 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10614#endif
10615 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10616
10617 /*
10618 * RIP and RFLAGS.
10619 */
10620 uint32_t u32Eflags;
10621#if HC_ARCH_BITS == 64
10622 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10623 AssertRCBreak(rc);
10624 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10625 if ( !fLongModeGuest
10626 || !pCtx->cs.Attr.n.u1Long)
10627 {
10628 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10629 }
10630 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10631 * must be identical if the "IA-32e mode guest" VM-entry
10632 * control is 1 and CS.L is 1. No check applies if the
10633 * CPU supports 64 linear-address bits. */
10634
10635 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10636 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10637 AssertRCBreak(rc);
10638 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10639 VMX_IGS_RFLAGS_RESERVED);
10640 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10641 u32Eflags = u64Val;
10642#else
10643 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10644 AssertRCBreak(rc);
10645 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10646 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10647#endif
10648
10649 if ( fLongModeGuest
10650 || ( fUnrestrictedGuest
10651 && !(u32GuestCr0 & X86_CR0_PE)))
10652 {
10653 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10654 }
10655
10656 uint32_t u32EntryInfo;
10657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10658 AssertRCBreak(rc);
10659 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10660 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10661 {
10662 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10663 }
10664
10665 /*
10666 * 64-bit checks.
10667 */
10668#if HC_ARCH_BITS == 64
10669 if (fLongModeGuest)
10670 {
10671 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10672 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10673 }
10674
10675 if ( !fLongModeGuest
10676 && (u32GuestCr4 & X86_CR4_PCIDE))
10677 {
10678 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10679 }
10680
10681 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10682 * 51:32 beyond the processor's physical-address width are 0. */
10683
10684 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10685 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10686 {
10687 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10688 }
10689
10690 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10691 AssertRCBreak(rc);
10692 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10693
10694 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10695 AssertRCBreak(rc);
10696 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10697#endif
10698
10699 /*
10700 * PERF_GLOBAL MSR.
10701 */
10702 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10703 {
10704 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10705 AssertRCBreak(rc);
10706 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10707 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10708 }
10709
10710 /*
10711 * PAT MSR.
10712 */
10713 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10714 {
10715 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10716 AssertRCBreak(rc);
10717 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10718 for (unsigned i = 0; i < 8; i++)
10719 {
10720 uint8_t u8Val = (u64Val & 0xff);
10721 if ( u8Val != 0 /* UC */
10722 && u8Val != 1 /* WC */
10723 && u8Val != 4 /* WT */
10724 && u8Val != 5 /* WP */
10725 && u8Val != 6 /* WB */
10726 && u8Val != 7 /* UC- */)
10727 {
10728 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10729 }
10730 u64Val >>= 8;
10731 }
10732 }
10733
10734 /*
10735 * EFER MSR.
10736 */
10737 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10738 {
10739 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10740 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10741 AssertRCBreak(rc);
10742 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10743 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10744 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10745 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10746 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10747 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10748 || !(u32GuestCr0 & X86_CR0_PG)
10749 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10750 VMX_IGS_EFER_LMA_LME_MISMATCH);
10751 }
10752
10753 /*
10754 * Segment registers.
10755 */
10756 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10757 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10758 if (!(u32Eflags & X86_EFL_VM))
10759 {
10760 /* CS */
10761 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10762 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10763 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10764 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10765 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10766 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10767 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10768 /* CS cannot be loaded with NULL in protected mode. */
10769 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10770 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10771 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10772 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10773 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10774 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10775 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10776 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10777 else
10778 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10779
10780 /* SS */
10781 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10782 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10783 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10784 if ( !(pCtx->cr0 & X86_CR0_PE)
10785 || pCtx->cs.Attr.n.u4Type == 3)
10786 {
10787 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10788 }
10789 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10790 {
10791 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10792 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10793 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10794 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10795 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10796 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10797 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10798 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10799 }
10800
10801 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10802 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10803 {
10804 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10805 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10806 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10807 || pCtx->ds.Attr.n.u4Type > 11
10808 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10809 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10810 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10811 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10812 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10813 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10814 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10815 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10816 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10817 }
10818 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10819 {
10820 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10821 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10822 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10823 || pCtx->es.Attr.n.u4Type > 11
10824 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10825 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10826 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10827 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10828 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10829 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10830 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10831 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10832 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10833 }
10834 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10835 {
10836 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10837 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10838 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10839 || pCtx->fs.Attr.n.u4Type > 11
10840 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10841 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10842 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10843 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10844 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10845 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10846 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10847 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10848 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10849 }
10850 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10851 {
10852 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10853 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10854 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10855 || pCtx->gs.Attr.n.u4Type > 11
10856 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10857 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10858 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10859 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10860 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10861 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10862 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10863 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10864 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10865 }
10866 /* 64-bit capable CPUs. */
10867#if HC_ARCH_BITS == 64
10868 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10869 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10870 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10871 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10872 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10873 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10874 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10875 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10876 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10877 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10878 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10879#endif
10880 }
10881 else
10882 {
10883 /* V86 mode checks. */
10884 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10885 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10886 {
10887 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10888 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10889 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10890 }
10891 else
10892 {
10893 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10894 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10895 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10896 }
10897
10898 /* CS */
10899 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10900 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10901 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10902 /* SS */
10903 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10904 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10905 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10906 /* DS */
10907 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10908 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10909 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10910 /* ES */
10911 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10912 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10913 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10914 /* FS */
10915 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10916 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10917 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10918 /* GS */
10919 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10920 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10921 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10922 /* 64-bit capable CPUs. */
10923#if HC_ARCH_BITS == 64
10924 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10925 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10926 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10927 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10928 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10929 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10930 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10931 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10932 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10933 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10934 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10935#endif
10936 }
10937
10938 /*
10939 * TR.
10940 */
10941 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10942 /* 64-bit capable CPUs. */
10943#if HC_ARCH_BITS == 64
10944 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10945#endif
10946 if (fLongModeGuest)
10947 {
10948 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10949 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10950 }
10951 else
10952 {
10953 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10954 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10955 VMX_IGS_TR_ATTR_TYPE_INVALID);
10956 }
10957 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10958 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10959 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10960 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10961 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10962 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10963 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10964 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10965
10966 /*
10967 * GDTR and IDTR.
10968 */
10969#if HC_ARCH_BITS == 64
10970 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10971 AssertRCBreak(rc);
10972 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10973
10974 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10975 AssertRCBreak(rc);
10976 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10977#endif
10978
10979 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10980 AssertRCBreak(rc);
10981 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10982
10983 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10984 AssertRCBreak(rc);
10985 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10986
10987 /*
10988 * Guest Non-Register State.
10989 */
10990 /* Activity State. */
10991 uint32_t u32ActivityState;
10992 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10993 AssertRCBreak(rc);
10994 HMVMX_CHECK_BREAK( !u32ActivityState
10995 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10996 VMX_IGS_ACTIVITY_STATE_INVALID);
10997 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10998 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10999 uint32_t u32IntrState;
11000 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
11001 AssertRCBreak(rc);
11002 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
11003 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11004 {
11005 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11006 }
11007
11008 /** @todo Activity state and injecting interrupts. Left as a todo since we
11009 * currently don't use activity states but ACTIVE. */
11010
11011 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11012 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11013
11014 /* Guest interruptibility-state. */
11015 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11016 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
11017 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11018 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11019 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11020 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11021 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11022 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
11023 {
11024 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
11025 {
11026 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11027 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11028 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11029 }
11030 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11031 {
11032 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11033 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11034 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11035 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11036 }
11037 }
11038 /** @todo Assumes the processor is not in SMM. */
11039 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11040 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11041 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11042 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11043 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11044 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
11045 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
11046 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11047 {
11048 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
11049 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11050 }
11051
11052 /* Pending debug exceptions. */
11053#if HC_ARCH_BITS == 64
11054 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11055 AssertRCBreak(rc);
11056 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11057 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11058 u32Val = u64Val; /* For pending debug exceptions checks below. */
11059#else
11060 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11061 AssertRCBreak(rc);
11062 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11063 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11064#endif
11065
11066 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11067 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11068 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11069 {
11070 if ( (u32Eflags & X86_EFL_TF)
11071 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11072 {
11073 /* Bit 14 is PendingDebug.BS. */
11074 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11075 }
11076 if ( !(u32Eflags & X86_EFL_TF)
11077 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11078 {
11079 /* Bit 14 is PendingDebug.BS. */
11080 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11081 }
11082 }
11083
11084 /* VMCS link pointer. */
11085 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11086 AssertRCBreak(rc);
11087 if (u64Val != UINT64_C(0xffffffffffffffff))
11088 {
11089 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11090 /** @todo Bits beyond the processor's physical-address width MBZ. */
11091 /** @todo 32-bit located in memory referenced by value of this field (as a
11092 * physical address) must contain the processor's VMCS revision ID. */
11093 /** @todo SMM checks. */
11094 }
11095
11096 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11097 * not using Nested Paging? */
11098 if ( pVM->hm.s.fNestedPaging
11099 && !fLongModeGuest
11100 && CPUMIsGuestInPAEModeEx(pCtx))
11101 {
11102 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11103 AssertRCBreak(rc);
11104 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11105
11106 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11107 AssertRCBreak(rc);
11108 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11109
11110 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11111 AssertRCBreak(rc);
11112 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11113
11114 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11115 AssertRCBreak(rc);
11116 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11117 }
11118
11119 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11120 if (uError == VMX_IGS_ERROR)
11121 uError = VMX_IGS_REASON_NOT_FOUND;
11122 } while (0);
11123
11124 pVCpu->hm.s.u32HMError = uError;
11125 return uError;
11126
11127#undef HMVMX_ERROR_BREAK
11128#undef HMVMX_CHECK_BREAK
11129}
11130
11131
11132/** @name VM-exit handlers.
11133 * @{
11134 */
11135/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11136/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11137/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11138
11139/**
11140 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11141 */
11142HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11143{
11144 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11145 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11146 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11147 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11148 return VINF_SUCCESS;
11149 return VINF_EM_RAW_INTERRUPT;
11150}
11151
11152
11153/**
11154 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11155 */
11156HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11157{
11158 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11159 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11160
11161 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11162 AssertRCReturn(rc, rc);
11163
11164 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11165 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11166 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11167 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11168
11169 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11170 {
11171 /*
11172 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11173 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11174 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11175 *
11176 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11177 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11178 */
11179 VMXDispatchHostNmi();
11180 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11181 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11182 return VINF_SUCCESS;
11183 }
11184
11185 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11186 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11187 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11188 { /* likely */ }
11189 else
11190 {
11191 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11192 rcStrictRc1 = VINF_SUCCESS;
11193 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11194 return rcStrictRc1;
11195 }
11196
11197 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11198 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11199 switch (uIntType)
11200 {
11201 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11202 Assert(uVector == X86_XCPT_DB);
11203 RT_FALL_THRU();
11204 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11205 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11206 RT_FALL_THRU();
11207 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11208 {
11209 /*
11210 * If there's any exception caused as a result of event injection, the resulting
11211 * secondary/final execption will be pending, we shall continue guest execution
11212 * after injecting the event. The page-fault case is complicated and we manually
11213 * handle any currently pending event in hmR0VmxExitXcptPF.
11214 */
11215 if (!pVCpu->hm.s.Event.fPending)
11216 { /* likely */ }
11217 else if (uVector != X86_XCPT_PF)
11218 {
11219 rc = VINF_SUCCESS;
11220 break;
11221 }
11222
11223 switch (uVector)
11224 {
11225 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11226 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11227 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11228 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11229 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11230 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11231
11232 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11233 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11234 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11235 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11236 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11237 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11238 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11239 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11240 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11241 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11242 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11243 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11244 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11245 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11246 default:
11247 {
11248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11249 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11250 {
11251 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11252 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11253 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11254
11255 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11256 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11257 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11258 AssertRCReturn(rc, rc);
11259 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11260 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11261 0 /* GCPtrFaultAddress */);
11262 }
11263 else
11264 {
11265 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11266 pVCpu->hm.s.u32HMError = uVector;
11267 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11268 }
11269 break;
11270 }
11271 }
11272 break;
11273 }
11274
11275 default:
11276 {
11277 pVCpu->hm.s.u32HMError = uExitIntInfo;
11278 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11279 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11280 break;
11281 }
11282 }
11283 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11284 return rc;
11285}
11286
11287
11288/**
11289 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11290 */
11291HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11292{
11293 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11294
11295 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11296 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11297
11298 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11299 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11300 return VINF_SUCCESS;
11301}
11302
11303
11304/**
11305 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11306 */
11307HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11308{
11309 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11310 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11311 {
11312 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11313 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11314 }
11315
11316 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11317
11318 /*
11319 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11320 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11321 */
11322 uint32_t fIntrState = 0;
11323 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11324 AssertRCReturn(rc, rc);
11325
11326 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11327 if ( fBlockSti
11328 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11329 {
11330 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11331 }
11332
11333 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11334 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11335
11336 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11337 return VINF_SUCCESS;
11338}
11339
11340
11341/**
11342 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11343 */
11344HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11345{
11346 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11347 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11348}
11349
11350
11351/**
11352 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11353 */
11354HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11355{
11356 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11357 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11358}
11359
11360
11361/**
11362 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11363 */
11364HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11365{
11366 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11367
11368 /*
11369 * Get the state we need and update the exit history entry.
11370 */
11371 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11372 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11373 AssertRCReturn(rc, rc);
11374
11375 VBOXSTRICTRC rcStrict;
11376 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11377 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11378 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11379 if (!pExitRec)
11380 {
11381 /*
11382 * Regular CPUID instruction execution.
11383 */
11384 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11385 if (rcStrict == VINF_SUCCESS)
11386 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RAX
11387 | HM_CHANGED_GUEST_RCX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RBX);
11388 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11389 {
11390 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11391 rcStrict = VINF_SUCCESS;
11392 }
11393 }
11394 else
11395 {
11396 /*
11397 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11398 */
11399 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11400 AssertRCReturn(rc2, rc2);
11401
11402 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11403 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11404
11405 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11406 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11407
11408 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11409 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11410 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11411 }
11412 return rcStrict;
11413}
11414
11415
11416/**
11417 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11418 */
11419HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11420{
11421 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11422 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11423 AssertRCReturn(rc, rc);
11424
11425 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11426 return VINF_EM_RAW_EMULATE_INSTR;
11427
11428 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11429 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11430}
11431
11432
11433/**
11434 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11435 */
11436HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11437{
11438 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11439 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11440 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11441 AssertRCReturn(rc, rc);
11442
11443 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11444 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11445 {
11446 /* If we get a spurious VM-exit when offsetting is enabled,
11447 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11448 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11449 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11450 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11451 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11452 }
11453 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11454 {
11455 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11456 rcStrict = VINF_SUCCESS;
11457 }
11458 return rcStrict;
11459}
11460
11461
11462/**
11463 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11464 */
11465HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11466{
11467 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11468 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11469 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11470 AssertRCReturn(rc, rc);
11471
11472 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11473 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11474 {
11475 /* If we get a spurious VM-exit when offsetting is enabled,
11476 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11477 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11478 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11479 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11480 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RCX);
11481 }
11482 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11483 {
11484 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11485 rcStrict = VINF_SUCCESS;
11486 }
11487 return rcStrict;
11488}
11489
11490
11491/**
11492 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11493 */
11494HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11495{
11496 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11497 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11498 AssertRCReturn(rc, rc);
11499
11500 PVM pVM = pVCpu->CTX_SUFF(pVM);
11501 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11502 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11503 if (RT_LIKELY(rc == VINF_SUCCESS))
11504 {
11505 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11506 Assert(pVmxTransient->cbInstr == 2);
11507 }
11508 else
11509 {
11510 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11511 rc = VERR_EM_INTERPRETER;
11512 }
11513 return rc;
11514}
11515
11516
11517/**
11518 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11519 */
11520HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11521{
11522 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11523
11524 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11525 if (EMAreHypercallInstructionsEnabled(pVCpu))
11526 {
11527 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11528 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11529 AssertRCReturn(rc, rc);
11530
11531 /* Perform the hypercall. */
11532 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11533 if (rcStrict == VINF_SUCCESS)
11534 {
11535 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11536 AssertRCReturn(rc, rc);
11537 }
11538 else
11539 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11540 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11541 || RT_FAILURE(rcStrict));
11542
11543 /* If the hypercall changes anything other than guest's general-purpose registers,
11544 we would need to reload the guest changed bits here before VM-entry. */
11545 }
11546 else
11547 Log4Func(("Hypercalls not enabled\n"));
11548
11549 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11550 if (RT_FAILURE(rcStrict))
11551 {
11552 hmR0VmxSetPendingXcptUD(pVCpu);
11553 rcStrict = VINF_SUCCESS;
11554 }
11555
11556 return rcStrict;
11557}
11558
11559
11560/**
11561 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11562 */
11563HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11564{
11565 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11566 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11567
11568 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11569 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11570 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11571 AssertRCReturn(rc, rc);
11572
11573 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11574
11575 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11576 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11577 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11578 {
11579 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11580 rcStrict = VINF_SUCCESS;
11581 }
11582 else
11583 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11584 VBOXSTRICTRC_VAL(rcStrict)));
11585 return rcStrict;
11586}
11587
11588
11589/**
11590 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11591 */
11592HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11593{
11594 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11595 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11596 AssertRCReturn(rc, rc);
11597
11598 PVM pVM = pVCpu->CTX_SUFF(pVM);
11599 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11600 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11601 if (RT_LIKELY(rc == VINF_SUCCESS))
11602 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11603 else
11604 {
11605 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11606 rc = VERR_EM_INTERPRETER;
11607 }
11608 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11609 return rc;
11610}
11611
11612
11613/**
11614 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11615 */
11616HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11617{
11618 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11619 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11620 AssertRCReturn(rc, rc);
11621
11622 PVM pVM = pVCpu->CTX_SUFF(pVM);
11623 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11624 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11625 rc = VBOXSTRICTRC_VAL(rc2);
11626 if (RT_LIKELY( rc == VINF_SUCCESS
11627 || rc == VINF_EM_HALT))
11628 {
11629 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11630 AssertRCReturn(rc3, rc3);
11631
11632 if ( rc == VINF_EM_HALT
11633 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11634 rc = VINF_SUCCESS;
11635 }
11636 else
11637 {
11638 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11639 rc = VERR_EM_INTERPRETER;
11640 }
11641 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11642 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11643 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11644 return rc;
11645}
11646
11647
11648/**
11649 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11650 */
11651HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11652{
11653 /*
11654 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11655 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11656 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11657 * VMX root operation. If we get here, something funny is going on.
11658 *
11659 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11660 */
11661 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11662 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11663 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11664}
11665
11666
11667/**
11668 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11669 */
11670HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11671{
11672 /*
11673 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11674 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11675 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11676 * an SMI. If we get here, something funny is going on.
11677 *
11678 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11679 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11680 */
11681 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11682 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11683 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11684}
11685
11686
11687/**
11688 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11689 */
11690HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11691{
11692 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11693 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11694 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11695 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11696}
11697
11698
11699/**
11700 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11701 */
11702HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11703{
11704 /*
11705 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11706 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11707 * See Intel spec. 25.3 "Other Causes of VM-exits".
11708 */
11709 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11710 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11711 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11712}
11713
11714
11715/**
11716 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11717 * VM-exit.
11718 */
11719HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11720{
11721 /*
11722 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11723 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11724 *
11725 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11726 * See Intel spec. "23.8 Restrictions on VMX operation".
11727 */
11728 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11729 return VINF_SUCCESS;
11730}
11731
11732
11733/**
11734 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11735 * VM-exit.
11736 */
11737HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11738{
11739 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11740 return VINF_EM_RESET;
11741}
11742
11743
11744/**
11745 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11746 */
11747HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11748{
11749 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11750 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11751
11752 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11753 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11754 AssertRCReturn(rc, rc);
11755
11756 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11757 rc = VINF_SUCCESS;
11758 else
11759 rc = VINF_EM_HALT;
11760
11761 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11762 if (rc != VINF_SUCCESS)
11763 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11764 return rc;
11765}
11766
11767
11768/**
11769 * VM-exit handler for instructions that result in a \#UD exception delivered to
11770 * the guest.
11771 */
11772HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11773{
11774 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11775 hmR0VmxSetPendingXcptUD(pVCpu);
11776 return VINF_SUCCESS;
11777}
11778
11779
11780/**
11781 * VM-exit handler for expiry of the VMX preemption timer.
11782 */
11783HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11784{
11785 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11786
11787 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11788 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11789
11790 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11791 PVM pVM = pVCpu->CTX_SUFF(pVM);
11792 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11793 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11794 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11795}
11796
11797
11798/**
11799 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11800 */
11801HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11802{
11803 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11804
11805 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11806 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11807 AssertRCReturn(rc, rc);
11808
11809 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11810 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11811 : HM_CHANGED_RAISED_XCPT_MASK);
11812
11813 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11814 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11815
11816 return rcStrict;
11817}
11818
11819
11820/**
11821 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11822 */
11823HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11824{
11825 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11826 /** @todo Use VM-exit instruction information. */
11827 return VERR_EM_INTERPRETER;
11828}
11829
11830
11831/**
11832 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11833 * Error VM-exit.
11834 */
11835HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11836{
11837 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11838 AssertRCReturn(rc, rc);
11839 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11840 if (RT_FAILURE(rc))
11841 return rc;
11842
11843 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11844 NOREF(uInvalidReason);
11845
11846#ifdef VBOX_STRICT
11847 uint32_t fIntrState;
11848 RTHCUINTREG uHCReg;
11849 uint64_t u64Val;
11850 uint32_t u32Val;
11851
11852 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11853 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11854 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11855 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11856 AssertRCReturn(rc, rc);
11857
11858 Log4(("uInvalidReason %u\n", uInvalidReason));
11859 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11860 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11861 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11862 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11863
11864 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11865 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11866 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11867 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11868 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11869 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11870 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11871 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11872 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11873 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11874 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11875 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11876
11877 hmR0DumpRegs(pVCpu);
11878#else
11879 NOREF(pVmxTransient);
11880#endif
11881
11882 return VERR_VMX_INVALID_GUEST_STATE;
11883}
11884
11885
11886/**
11887 * VM-exit handler for VM-entry failure due to an MSR-load
11888 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11889 */
11890HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11891{
11892 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11893 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11894}
11895
11896
11897/**
11898 * VM-exit handler for VM-entry failure due to a machine-check event
11899 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11900 */
11901HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11902{
11903 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11904 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11905}
11906
11907
11908/**
11909 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11910 * theory.
11911 */
11912HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11913{
11914 RT_NOREF2(pVCpu, pVmxTransient);
11915 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11916 return VERR_VMX_UNDEFINED_EXIT_CODE;
11917}
11918
11919
11920/**
11921 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11922 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11923 * Conditional VM-exit.
11924 */
11925HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11926{
11927 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11928
11929 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11930 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11931 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11932 return VERR_EM_INTERPRETER;
11933 AssertMsgFailed(("Unexpected XDTR access\n"));
11934 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11935}
11936
11937
11938/**
11939 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11940 */
11941HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11942{
11943 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11944
11945 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11946 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11947 return VERR_EM_INTERPRETER;
11948 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11949 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11950}
11951
11952
11953/**
11954 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11955 */
11956HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11957{
11958 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11959
11960 /** @todo Optimize this: We currently drag in in the whole MSR state
11961 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11962 * MSRs required. That would require changes to IEM and possibly CPUM too.
11963 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11964 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11965 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11966 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11967 AssertRCReturn(rc, rc);
11968
11969 Log4Func(("ecx=%#RX32\n", idMsr));
11970
11971#ifdef VBOX_STRICT
11972 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11973 {
11974 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11975 && idMsr != MSR_K6_EFER)
11976 {
11977 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11978 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11979 }
11980 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11981 {
11982 VMXMSREXITREAD enmRead;
11983 VMXMSREXITWRITE enmWrite;
11984 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11985 AssertRCReturn(rc2, rc2);
11986 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11987 {
11988 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11989 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11990 }
11991 }
11992 }
11993#endif
11994
11995 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11996 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11997 if (rcStrict == VINF_SUCCESS)
11998 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11999 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
12000 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12001 {
12002 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12003 rcStrict = VINF_SUCCESS;
12004 }
12005 else
12006 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12007
12008 return rcStrict;
12009}
12010
12011
12012/**
12013 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12014 */
12015HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12016{
12017 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12018
12019 /** @todo Optimize this: We currently drag in in the whole MSR state
12020 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
12021 * MSRs required. That would require changes to IEM and possibly CPUM too.
12022 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
12023 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
12024 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12025 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
12026 AssertRCReturn(rc, rc);
12027
12028 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
12029
12030 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
12031 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12032
12033 if (rcStrict == VINF_SUCCESS)
12034 {
12035 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12036
12037 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12038 if ( idMsr == MSR_IA32_APICBASE
12039 || ( idMsr >= MSR_IA32_X2APIC_START
12040 && idMsr <= MSR_IA32_X2APIC_END))
12041 {
12042 /*
12043 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12044 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
12045 */
12046 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12047 }
12048 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12049 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12050 else if (idMsr == MSR_K6_EFER)
12051 {
12052 /*
12053 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12054 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12055 * the other bits as well, SCE and NXE. See @bugref{7368}.
12056 */
12057 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
12058 | HM_CHANGED_VMX_EXIT_CTLS);
12059 }
12060
12061 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12062 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12063 {
12064 switch (idMsr)
12065 {
12066 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12067 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12068 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12069 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12070 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12071 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12072 default:
12073 {
12074 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12075 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12076 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12077 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12078 break;
12079 }
12080 }
12081 }
12082#ifdef VBOX_STRICT
12083 else
12084 {
12085 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12086 switch (idMsr)
12087 {
12088 case MSR_IA32_SYSENTER_CS:
12089 case MSR_IA32_SYSENTER_EIP:
12090 case MSR_IA32_SYSENTER_ESP:
12091 case MSR_K8_FS_BASE:
12092 case MSR_K8_GS_BASE:
12093 {
12094 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12095 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12096 }
12097
12098 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12099 default:
12100 {
12101 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12102 {
12103 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12104 if (idMsr != MSR_K6_EFER)
12105 {
12106 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12107 idMsr));
12108 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12109 }
12110 }
12111
12112 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12113 {
12114 VMXMSREXITREAD enmRead;
12115 VMXMSREXITWRITE enmWrite;
12116 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
12117 AssertRCReturn(rc2, rc2);
12118 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12119 {
12120 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12121 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12122 }
12123 }
12124 break;
12125 }
12126 }
12127 }
12128#endif /* VBOX_STRICT */
12129 }
12130 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12131 {
12132 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12133 rcStrict = VINF_SUCCESS;
12134 }
12135 else
12136 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12137
12138 return rcStrict;
12139}
12140
12141
12142/**
12143 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12144 */
12145HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12146{
12147 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12148 /** @todo The guest has likely hit a contended spinlock. We might want to
12149 * poke a schedule different guest VCPU. */
12150 return VINF_EM_RAW_INTERRUPT;
12151}
12152
12153
12154/**
12155 * VM-exit handler for when the TPR value is lowered below the specified
12156 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12157 */
12158HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12159{
12160 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12161 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12162
12163 /*
12164 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12165 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12166 */
12167 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12168 return VINF_SUCCESS;
12169}
12170
12171
12172/**
12173 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12174 * VM-exit.
12175 *
12176 * @retval VINF_SUCCESS when guest execution can continue.
12177 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12178 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12179 * interpreter.
12180 */
12181HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12182{
12183 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12184 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12185
12186 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12187 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12188 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12189 AssertRCReturn(rc, rc);
12190
12191 VBOXSTRICTRC rcStrict;
12192 PVM pVM = pVCpu->CTX_SUFF(pVM);
12193 RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12194 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12195 switch (uAccessType)
12196 {
12197 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12198 {
12199 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12200 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12201 VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12202 AssertMsg( rcStrict == VINF_SUCCESS
12203 || rcStrict == VINF_IEM_RAISED_XCPT
12204 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12205
12206 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12207 {
12208 case 0:
12209 {
12210 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12211 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12212 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12213 Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12214
12215 /*
12216 * This is a kludge for handling switches back to real mode when we try to use
12217 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
12218 * deal with special selector values, so we have to return to ring-3 and run
12219 * there till the selector values are V86 mode compatible.
12220 *
12221 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12222 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12223 * at the end of this function.
12224 */
12225 if ( rc == VINF_SUCCESS
12226 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12227 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12228 && (uOldCr0 & X86_CR0_PE)
12229 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12230 {
12231 /** @todo check selectors rather than returning all the time. */
12232 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12233 rcStrict = VINF_EM_RESCHEDULE_REM;
12234 }
12235 break;
12236 }
12237
12238 case 2:
12239 {
12240 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12241 /* Nothing to do here, CR2 it's not part of the VMCS. */
12242 break;
12243 }
12244
12245 case 3:
12246 {
12247 Assert( !pVM->hm.s.fNestedPaging
12248 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12249 || pVCpu->hm.s.fUsingDebugLoop);
12250 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12251 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12252 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
12253 Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12254 break;
12255 }
12256
12257 case 4:
12258 {
12259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12260 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12261 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
12262 Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12263 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12264 break;
12265 }
12266
12267 case 8:
12268 {
12269 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12270 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12271 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12272 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
12273 break;
12274 }
12275 default:
12276 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12277 break;
12278 }
12279 break;
12280 }
12281
12282 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12283 {
12284 Assert( !pVM->hm.s.fNestedPaging
12285 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12286 || pVCpu->hm.s.fUsingDebugLoop
12287 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12288 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12289 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12290 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12291
12292 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12293 VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12294 AssertMsg( rcStrict == VINF_SUCCESS
12295 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12296#ifdef VBOX_WITH_STATISTICS
12297 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12298 {
12299 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12300 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12301 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12302 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12303 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12304 }
12305#endif
12306 Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12307 VBOXSTRICTRC_VAL(rcStrict)));
12308 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12309 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12310 else
12311 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12312 break;
12313 }
12314
12315 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12316 {
12317 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12318 AssertMsg( rcStrict == VINF_SUCCESS
12319 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12320
12321 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12322 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12323 Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12324 break;
12325 }
12326
12327 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12328 {
12329 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12330 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual));
12331 AssertMsg( rcStrict == VINF_SUCCESS
12332 || rcStrict == VINF_IEM_RAISED_XCPT
12333 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12334
12335 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12336 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12337 Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12338 break;
12339 }
12340
12341 default:
12342 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12343 VERR_VMX_UNEXPECTED_EXCEPTION);
12344 }
12345
12346 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12347 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12348 if (rcStrict == VINF_IEM_RAISED_XCPT)
12349 {
12350 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12351 rcStrict = VINF_SUCCESS;
12352 }
12353
12354 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12355 NOREF(pVM);
12356 return rcStrict;
12357}
12358
12359
12360/**
12361 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12362 * VM-exit.
12363 */
12364HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12365{
12366 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12367 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12368
12369 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12370 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12371 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12372 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12373 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12374 AssertRCReturn(rc, rc);
12375
12376 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12377 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12378 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12379 bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12380 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12381 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12382 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12383 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12384
12385 /*
12386 * Update exit history to see if this exit can be optimized.
12387 */
12388 VBOXSTRICTRC rcStrict;
12389 PCEMEXITREC pExitRec = NULL;
12390 if ( !fGstStepping
12391 && !fDbgStepping)
12392 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12393 !fIOString
12394 ? !fIOWrite
12395 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12396 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12397 : !fIOWrite
12398 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12399 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12400 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12401 if (!pExitRec)
12402 {
12403 /* I/O operation lookup arrays. */
12404 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12405 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12406 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12407 uint32_t const cbInstr = pVmxTransient->cbInstr;
12408 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12409 PVM pVM = pVCpu->CTX_SUFF(pVM);
12410 if (fIOString)
12411 {
12412 /*
12413 * INS/OUTS - I/O String instruction.
12414 *
12415 * Use instruction-information if available, otherwise fall back on
12416 * interpreting the instruction.
12417 */
12418 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12419 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12420 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12421 if (fInsOutsInfo)
12422 {
12423 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12424 AssertRCReturn(rc2, rc2);
12425 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12426 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12427 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12428 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12429 if (fIOWrite)
12430 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12431 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12432 else
12433 {
12434 /*
12435 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12436 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12437 * See Intel Instruction spec. for "INS".
12438 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12439 */
12440 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12441 }
12442 }
12443 else
12444 rcStrict = IEMExecOne(pVCpu);
12445
12446 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12447 fUpdateRipAlready = true;
12448 }
12449 else
12450 {
12451 /*
12452 * IN/OUT - I/O instruction.
12453 */
12454 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12455 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12456 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12457 if (fIOWrite)
12458 {
12459 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12460 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12461 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12462 && !pCtx->eflags.Bits.u1TF)
12463 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12464 }
12465 else
12466 {
12467 uint32_t u32Result = 0;
12468 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12469 if (IOM_SUCCESS(rcStrict))
12470 {
12471 /* Save result of I/O IN instr. in AL/AX/EAX. */
12472 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12473 }
12474 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12475 && !pCtx->eflags.Bits.u1TF)
12476 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12477 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12478 }
12479 }
12480
12481 if (IOM_SUCCESS(rcStrict))
12482 {
12483 if (!fUpdateRipAlready)
12484 {
12485 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12486 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12487 }
12488
12489 /*
12490 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12491 * while booting Fedora 17 64-bit guest.
12492 *
12493 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12494 */
12495 if (fIOString)
12496 {
12497 /** @todo Single-step for INS/OUTS with REP prefix? */
12498 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12499 }
12500 else if ( !fDbgStepping
12501 && fGstStepping)
12502 {
12503 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12504 AssertRCReturn(rc, rc);
12505 }
12506
12507 /*
12508 * If any I/O breakpoints are armed, we need to check if one triggered
12509 * and take appropriate action.
12510 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12511 */
12512 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12513 AssertRCReturn(rc, rc);
12514
12515 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12516 * execution engines about whether hyper BPs and such are pending. */
12517 uint32_t const uDr7 = pCtx->dr[7];
12518 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12519 && X86_DR7_ANY_RW_IO(uDr7)
12520 && (pCtx->cr4 & X86_CR4_DE))
12521 || DBGFBpIsHwIoArmed(pVM)))
12522 {
12523 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12524
12525 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12526 VMMRZCallRing3Disable(pVCpu);
12527 HM_DISABLE_PREEMPT(pVCpu);
12528
12529 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12530
12531 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12532 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12533 {
12534 /* Raise #DB. */
12535 if (fIsGuestDbgActive)
12536 ASMSetDR6(pCtx->dr[6]);
12537 if (pCtx->dr[7] != uDr7)
12538 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12539
12540 hmR0VmxSetPendingXcptDB(pVCpu);
12541 }
12542 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12543 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12544 else if ( rcStrict2 != VINF_SUCCESS
12545 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12546 rcStrict = rcStrict2;
12547 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12548
12549 HM_RESTORE_PREEMPT();
12550 VMMRZCallRing3Enable(pVCpu);
12551 }
12552 }
12553
12554#ifdef VBOX_STRICT
12555 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12556 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12557 Assert(!fIOWrite);
12558 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12559 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12560 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12561 Assert(fIOWrite);
12562 else
12563 {
12564# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12565 * statuses, that the VMM device and some others may return. See
12566 * IOM_SUCCESS() for guidance. */
12567 AssertMsg( RT_FAILURE(rcStrict)
12568 || rcStrict == VINF_SUCCESS
12569 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12570 || rcStrict == VINF_EM_DBG_BREAKPOINT
12571 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12572 || rcStrict == VINF_EM_RAW_TO_R3
12573 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12574# endif
12575 }
12576#endif
12577 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12578 }
12579 else
12580 {
12581 /*
12582 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12583 */
12584 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12585 AssertRCReturn(rc2, rc2);
12586 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12587 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12588 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12589 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12590 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12591 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12592
12593 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12594 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12595
12596 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12597 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12598 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12599 }
12600 return rcStrict;
12601}
12602
12603
12604/**
12605 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12606 * VM-exit.
12607 */
12608HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12609{
12610 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12611
12612 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12613 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12614 AssertRCReturn(rc, rc);
12615 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12616 {
12617 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12618 AssertRCReturn(rc, rc);
12619 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12620 {
12621 uint32_t uErrCode;
12622 RTGCUINTPTR GCPtrFaultAddress;
12623 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12624 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12625 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12626 if (fErrorCodeValid)
12627 {
12628 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12629 AssertRCReturn(rc, rc);
12630 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12631 }
12632 else
12633 uErrCode = 0;
12634
12635 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12636 && uVector == X86_XCPT_PF)
12637 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12638 else
12639 GCPtrFaultAddress = 0;
12640
12641 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12642 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12643
12644 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12645 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12646 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12647 }
12648 }
12649
12650 /* Fall back to the interpreter to emulate the task-switch. */
12651 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12652 return VERR_EM_INTERPRETER;
12653}
12654
12655
12656/**
12657 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12658 */
12659HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12660{
12661 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12662 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12663 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12664 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12665 AssertRCReturn(rc, rc);
12666 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12667 return VINF_EM_DBG_STEPPED;
12668}
12669
12670
12671/**
12672 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12673 */
12674HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12675{
12676 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12677
12678 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12679
12680 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12681 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12682 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12683 {
12684 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12685 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12686 {
12687 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12688 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12689 }
12690 }
12691 else
12692 {
12693 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12694 rcStrict1 = VINF_SUCCESS;
12695 return rcStrict1;
12696 }
12697
12698 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12699 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12700 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12701 AssertRCReturn(rc, rc);
12702
12703 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12704 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12705 VBOXSTRICTRC rcStrict2;
12706 switch (uAccessType)
12707 {
12708 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12709 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12710 {
12711 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12712 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12713 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12714
12715 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12716 GCPhys &= PAGE_BASE_GC_MASK;
12717 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12718 PVM pVM = pVCpu->CTX_SUFF(pVM);
12719 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12720 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12721
12722 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12723 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12724 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12725 CPUMCTX2CORE(pCtx), GCPhys);
12726 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12727 if ( rcStrict2 == VINF_SUCCESS
12728 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12729 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12730 {
12731 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12732 | HM_CHANGED_GUEST_APIC_TPR);
12733 rcStrict2 = VINF_SUCCESS;
12734 }
12735 break;
12736 }
12737
12738 default:
12739 Log4Func(("uAccessType=%#x\n", uAccessType));
12740 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12741 break;
12742 }
12743
12744 if (rcStrict2 != VINF_SUCCESS)
12745 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12746 return rcStrict2;
12747}
12748
12749
12750/**
12751 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12752 * VM-exit.
12753 */
12754HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12755{
12756 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12757
12758 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12759 if (pVmxTransient->fWasGuestDebugStateActive)
12760 {
12761 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12762 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12763 }
12764
12765 if ( !pVCpu->hm.s.fSingleInstruction
12766 && !pVmxTransient->fWasHyperDebugStateActive)
12767 {
12768 Assert(!DBGFIsStepping(pVCpu));
12769 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12770
12771 /* Don't intercept MOV DRx any more. */
12772 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12773 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12774 AssertRCReturn(rc, rc);
12775
12776 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12777 VMMRZCallRing3Disable(pVCpu);
12778 HM_DISABLE_PREEMPT(pVCpu);
12779
12780 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12781 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12782 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12783
12784 HM_RESTORE_PREEMPT();
12785 VMMRZCallRing3Enable(pVCpu);
12786
12787#ifdef VBOX_WITH_STATISTICS
12788 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12789 AssertRCReturn(rc, rc);
12790 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12791 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12792 else
12793 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12794#endif
12795 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12796 return VINF_SUCCESS;
12797 }
12798
12799 /*
12800 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12801 * Update the segment registers and DR7 from the CPU.
12802 */
12803 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12804 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12805 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12806 AssertRCReturn(rc, rc);
12807 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12808
12809 PVM pVM = pVCpu->CTX_SUFF(pVM);
12810 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12811 {
12812 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12813 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12814 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12815 if (RT_SUCCESS(rc))
12816 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12817 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12818 }
12819 else
12820 {
12821 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12822 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12823 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12824 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12825 }
12826
12827 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12828 if (RT_SUCCESS(rc))
12829 {
12830 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12831 AssertRCReturn(rc2, rc2);
12832 return VINF_SUCCESS;
12833 }
12834 return rc;
12835}
12836
12837
12838/**
12839 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12840 * Conditional VM-exit.
12841 */
12842HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12843{
12844 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12845 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12846
12847 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12848 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12849 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12850 {
12851 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12852 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12853 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12854 {
12855 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12856 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12857 }
12858 }
12859 else
12860 {
12861 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12862 rcStrict1 = VINF_SUCCESS;
12863 return rcStrict1;
12864 }
12865
12866 /*
12867 * Get sufficent state and update the exit history entry.
12868 */
12869 RTGCPHYS GCPhys;
12870 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12871 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12872 AssertRCReturn(rc, rc);
12873
12874 VBOXSTRICTRC rcStrict;
12875 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12876 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12877 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12878 if (!pExitRec)
12879 {
12880 /*
12881 * If we succeed, resume guest execution.
12882 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12883 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12884 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12885 * weird case. See @bugref{6043}.
12886 */
12887 PVM pVM = pVCpu->CTX_SUFF(pVM);
12888 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12889 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12890 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12891 if ( rcStrict == VINF_SUCCESS
12892 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12893 || rcStrict == VERR_PAGE_NOT_PRESENT)
12894 {
12895 /* Successfully handled MMIO operation. */
12896 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12897 | HM_CHANGED_GUEST_APIC_TPR);
12898 rcStrict = VINF_SUCCESS;
12899 }
12900 }
12901 else
12902 {
12903 /*
12904 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12905 */
12906 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12907 AssertRCReturn(rc2, rc2);
12908
12909 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12910 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12911
12912 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12913 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12914
12915 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12916 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12917 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12918 }
12919 return VBOXSTRICTRC_TODO(rcStrict);
12920}
12921
12922
12923/**
12924 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12925 * VM-exit.
12926 */
12927HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12928{
12929 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12930 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12931
12932 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12933 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12934 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12935 {
12936 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12937 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12938 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12939 }
12940 else
12941 {
12942 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12943 rcStrict1 = VINF_SUCCESS;
12944 return rcStrict1;
12945 }
12946
12947 RTGCPHYS GCPhys;
12948 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12949 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12950 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12951 AssertRCReturn(rc, rc);
12952
12953 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12954 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12955
12956 RTGCUINT uErrorCode = 0;
12957 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12958 uErrorCode |= X86_TRAP_PF_ID;
12959 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12960 uErrorCode |= X86_TRAP_PF_RW;
12961 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12962 uErrorCode |= X86_TRAP_PF_P;
12963
12964 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12965
12966
12967 /* Handle the pagefault trap for the nested shadow table. */
12968 PVM pVM = pVCpu->CTX_SUFF(pVM);
12969 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12970
12971 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12972 pCtx->cs.Sel, pCtx->rip));
12973
12974 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12975 TRPMResetTrap(pVCpu);
12976
12977 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12978 if ( rcStrict2 == VINF_SUCCESS
12979 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12980 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12981 {
12982 /* Successfully synced our nested page tables. */
12983 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12984 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12985 return VINF_SUCCESS;
12986 }
12987
12988 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12989 return rcStrict2;
12990}
12991
12992/** @} */
12993
12994/** @name VM-exit exception handlers.
12995 * @{
12996 */
12997/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12998/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12999/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13000
13001/**
13002 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13003 */
13004static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13005{
13006 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13007 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13008
13009 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
13010 AssertRCReturn(rc, rc);
13011
13012 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13013 {
13014 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13015 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13016
13017 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13018 * provides VM-exit instruction length. If this causes problem later,
13019 * disassemble the instruction like it's done on AMD-V. */
13020 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13021 AssertRCReturn(rc2, rc2);
13022 return rc;
13023 }
13024
13025 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13026 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13027 return rc;
13028}
13029
13030
13031/**
13032 * VM-exit exception handler for \#BP (Breakpoint exception).
13033 */
13034static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13035{
13036 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13037 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13038
13039 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13040 AssertRCReturn(rc, rc);
13041
13042 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13043 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13044 if (rc == VINF_EM_RAW_GUEST_TRAP)
13045 {
13046 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13047 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13048 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13049 AssertRCReturn(rc, rc);
13050
13051 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13052 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13053 }
13054
13055 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13056 return rc;
13057}
13058
13059
13060/**
13061 * VM-exit exception handler for \#AC (alignment check exception).
13062 */
13063static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13064{
13065 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13066
13067 /*
13068 * Re-inject it. We'll detect any nesting before getting here.
13069 */
13070 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13071 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13072 AssertRCReturn(rc, rc);
13073 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13074
13075 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13076 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13077 return VINF_SUCCESS;
13078}
13079
13080
13081/**
13082 * VM-exit exception handler for \#DB (Debug exception).
13083 */
13084static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13085{
13086 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13087 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13088
13089 /*
13090 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13091 * for processing.
13092 */
13093 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13094
13095 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13096 uint64_t uDR6 = X86_DR6_INIT_VAL;
13097 uDR6 |= (pVmxTransient->uExitQual & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13098
13099 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13100 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13101 Log6Func(("rc=%Rrc\n", rc));
13102 if (rc == VINF_EM_RAW_GUEST_TRAP)
13103 {
13104 /*
13105 * The exception was for the guest. Update DR6, DR7.GD and
13106 * IA32_DEBUGCTL.LBR before forwarding it.
13107 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13108 */
13109 VMMRZCallRing3Disable(pVCpu);
13110 HM_DISABLE_PREEMPT(pVCpu);
13111
13112 pCtx->dr[6] &= ~X86_DR6_B_MASK;
13113 pCtx->dr[6] |= uDR6;
13114 if (CPUMIsGuestDebugStateActive(pVCpu))
13115 ASMSetDR6(pCtx->dr[6]);
13116
13117 HM_RESTORE_PREEMPT();
13118 VMMRZCallRing3Enable(pVCpu);
13119
13120 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
13121 AssertRCReturn(rc, rc);
13122
13123 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13124 pCtx->dr[7] &= ~X86_DR7_GD;
13125
13126 /* Paranoia. */
13127 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13128 pCtx->dr[7] |= X86_DR7_RA1_MASK;
13129
13130 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13131 AssertRCReturn(rc, rc);
13132
13133 /*
13134 * Raise #DB in the guest.
13135 *
13136 * It is important to reflect exactly what the VM-exit gave us (preserving the
13137 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13138 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13139 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13140 *
13141 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13142 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13143 */
13144 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13145 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13146 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13147 AssertRCReturn(rc, rc);
13148 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13149 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13150 return VINF_SUCCESS;
13151 }
13152
13153 /*
13154 * Not a guest trap, must be a hypervisor related debug event then.
13155 * Update DR6 in case someone is interested in it.
13156 */
13157 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13158 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13159 CPUMSetHyperDR6(pVCpu, uDR6);
13160
13161 return rc;
13162}
13163
13164/**
13165 * VM-exit exception handler for \#GP (General-protection exception).
13166 *
13167 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13168 */
13169static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13170{
13171 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13172 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13173
13174 int rc;
13175 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13176 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13177 { /* likely */ }
13178 else
13179 {
13180#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13181 Assert(pVCpu->hm.s.fUsingDebugLoop);
13182#endif
13183 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13184 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13185 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13186 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13187 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13188 AssertRCReturn(rc, rc);
13189 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13190 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13191 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13192 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13193 return rc;
13194 }
13195
13196 Assert(CPUMIsGuestInRealModeEx(pCtx));
13197 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13198
13199 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13200 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13201 AssertRCReturn(rc, rc);
13202
13203 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13204 uint32_t cbOp = 0;
13205 PVM pVM = pVCpu->CTX_SUFF(pVM);
13206 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13207 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13208 if (RT_SUCCESS(rc))
13209 {
13210 rc = VINF_SUCCESS;
13211 Assert(cbOp == pDis->cbInstr);
13212 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pCtx->cs.Sel, pCtx->rip));
13213 switch (pDis->pCurInstr->uOpcode)
13214 {
13215 case OP_CLI:
13216 {
13217 pCtx->eflags.Bits.u1IF = 0;
13218 pCtx->eflags.Bits.u1RF = 0;
13219 pCtx->rip += pDis->cbInstr;
13220 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13221 if ( !fDbgStepping
13222 && pCtx->eflags.Bits.u1TF)
13223 {
13224 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13225 AssertRCReturn(rc, rc);
13226 }
13227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13228 break;
13229 }
13230
13231 case OP_STI:
13232 {
13233 bool fOldIF = pCtx->eflags.Bits.u1IF;
13234 pCtx->eflags.Bits.u1IF = 1;
13235 pCtx->eflags.Bits.u1RF = 0;
13236 pCtx->rip += pDis->cbInstr;
13237 if (!fOldIF)
13238 {
13239 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
13240 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13241 }
13242 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13243 if ( !fDbgStepping
13244 && pCtx->eflags.Bits.u1TF)
13245 {
13246 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13247 AssertRCReturn(rc, rc);
13248 }
13249 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13250 break;
13251 }
13252
13253 case OP_HLT:
13254 {
13255 rc = VINF_EM_HALT;
13256 pCtx->rip += pDis->cbInstr;
13257 pCtx->eflags.Bits.u1RF = 0;
13258 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13260 break;
13261 }
13262
13263 case OP_POPF:
13264 {
13265 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pCtx->cs.Sel, pCtx->rip));
13266 uint32_t cbParm;
13267 uint32_t uMask;
13268 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
13269 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13270 {
13271 cbParm = 4;
13272 uMask = 0xffffffff;
13273 }
13274 else
13275 {
13276 cbParm = 2;
13277 uMask = 0xffff;
13278 }
13279
13280 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13281 RTGCPTR GCPtrStack = 0;
13282 X86EFLAGS Eflags;
13283 Eflags.u32 = 0;
13284 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13285 &GCPtrStack);
13286 if (RT_SUCCESS(rc))
13287 {
13288 Assert(sizeof(Eflags.u32) >= cbParm);
13289 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13290 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13291 }
13292 if (RT_FAILURE(rc))
13293 {
13294 rc = VERR_EM_INTERPRETER;
13295 break;
13296 }
13297 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pCtx->rsp, uMask, pCtx->rip));
13298 pCtx->eflags.u32 = (pCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13299 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13300 pCtx->esp += cbParm;
13301 pCtx->esp &= uMask;
13302 pCtx->rip += pDis->cbInstr;
13303 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13304 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13305 POPF restores EFLAGS.TF. */
13306 if ( !fDbgStepping
13307 && fGstStepping)
13308 {
13309 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13310 AssertRCReturn(rc, rc);
13311 }
13312 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13313 break;
13314 }
13315
13316 case OP_PUSHF:
13317 {
13318 uint32_t cbParm;
13319 uint32_t uMask;
13320 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13321 {
13322 cbParm = 4;
13323 uMask = 0xffffffff;
13324 }
13325 else
13326 {
13327 cbParm = 2;
13328 uMask = 0xffff;
13329 }
13330
13331 /* Get the stack pointer & push the contents of eflags onto the stack. */
13332 RTGCPTR GCPtrStack = 0;
13333 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask,
13334 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13335 if (RT_FAILURE(rc))
13336 {
13337 rc = VERR_EM_INTERPRETER;
13338 break;
13339 }
13340 X86EFLAGS Eflags = pCtx->eflags;
13341 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13342 Eflags.Bits.u1RF = 0;
13343 Eflags.Bits.u1VM = 0;
13344
13345 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13346 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13347 {
13348 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13349 rc = VERR_EM_INTERPRETER;
13350 break;
13351 }
13352 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13353 pCtx->esp -= cbParm;
13354 pCtx->esp &= uMask;
13355 pCtx->rip += pDis->cbInstr;
13356 pCtx->eflags.Bits.u1RF = 0;
13357 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13358 if ( !fDbgStepping
13359 && pCtx->eflags.Bits.u1TF)
13360 {
13361 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13362 AssertRCReturn(rc, rc);
13363 }
13364 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13365 break;
13366 }
13367
13368 case OP_IRET:
13369 {
13370 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13371 * instruction reference. */
13372 RTGCPTR GCPtrStack = 0;
13373 uint32_t uMask = 0xffff;
13374 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
13375 uint16_t aIretFrame[3];
13376 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13377 {
13378 rc = VERR_EM_INTERPRETER;
13379 break;
13380 }
13381 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13382 &GCPtrStack);
13383 if (RT_SUCCESS(rc))
13384 {
13385 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13386 PGMACCESSORIGIN_HM));
13387 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13388 }
13389 if (RT_FAILURE(rc))
13390 {
13391 rc = VERR_EM_INTERPRETER;
13392 break;
13393 }
13394 pCtx->eip = 0;
13395 pCtx->ip = aIretFrame[0];
13396 pCtx->cs.Sel = aIretFrame[1];
13397 pCtx->cs.ValidSel = aIretFrame[1];
13398 pCtx->cs.u64Base = (uint64_t)pCtx->cs.Sel << 4;
13399 pCtx->eflags.u32 = (pCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13400 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13401 pCtx->sp += sizeof(aIretFrame);
13402 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
13403 | HM_CHANGED_GUEST_CS);
13404 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13405 if ( !fDbgStepping
13406 && fGstStepping)
13407 {
13408 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13409 AssertRCReturn(rc, rc);
13410 }
13411 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pCtx->cs.Sel, pCtx->ip));
13412 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13413 break;
13414 }
13415
13416 case OP_INT:
13417 {
13418 uint16_t uVector = pDis->Param1.uValue & 0xff;
13419 hmR0VmxSetPendingIntN(pVCpu, uVector, pDis->cbInstr);
13420 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13421 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13422 break;
13423 }
13424
13425 case OP_INTO:
13426 {
13427 if (pCtx->eflags.Bits.u1OF)
13428 {
13429 hmR0VmxSetPendingXcptOF(pVCpu, pDis->cbInstr);
13430 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13431 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13432 }
13433 else
13434 {
13435 pCtx->eflags.Bits.u1RF = 0;
13436 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13437 }
13438 break;
13439 }
13440
13441 default:
13442 {
13443 pCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13444 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0 /* pvFault */,
13445 EMCODETYPE_SUPERVISOR);
13446 rc = VBOXSTRICTRC_VAL(rc2);
13447 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13448 /** @todo We have to set pending-debug exceptions here when the guest is
13449 * single-stepping depending on the instruction that was interpreted. */
13450
13451 /*
13452 * HACK ALERT! Detect mode change and go to ring-3 to properly exit this
13453 * real mode emulation stuff.
13454 */
13455 if ( rc == VINF_SUCCESS
13456 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
13457 {
13458 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13459 /** @todo Exit fRealOnV86Active here w/o dropping back to ring-3. */
13460 rc = VINF_EM_RESCHEDULE;
13461 }
13462
13463 Log4Func(("#GP rc=%Rrc\n", rc));
13464 break;
13465 }
13466 }
13467 }
13468 else
13469 rc = VERR_EM_INTERPRETER;
13470
13471 AssertMsg( rc == VINF_SUCCESS
13472 || rc == VERR_EM_INTERPRETER
13473 || rc == VINF_EM_HALT
13474 || rc == VINF_EM_RESCHEDULE
13475 , ("#GP Unexpected rc=%Rrc\n", rc));
13476 return rc;
13477}
13478
13479
13480/**
13481 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13482 * the exception reported in the VMX transient structure back into the VM.
13483 *
13484 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13485 * up-to-date.
13486 */
13487static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13488{
13489 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13490#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13491 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13492 ("uVector=%#x u32XcptBitmap=%#X32\n",
13493 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13494#endif
13495
13496 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13497 hmR0VmxCheckExitDueToEventDelivery(). */
13498 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13499 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13500 AssertRCReturn(rc, rc);
13501 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13502
13503#ifdef DEBUG_ramshankar
13504 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13505 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13506 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13507#endif
13508
13509 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13510 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13511 return VINF_SUCCESS;
13512}
13513
13514
13515/**
13516 * VM-exit exception handler for \#PF (Page-fault exception).
13517 */
13518static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13519{
13520 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13521 PVM pVM = pVCpu->CTX_SUFF(pVM);
13522 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13523 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13524 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13525 AssertRCReturn(rc, rc);
13526
13527 if (!pVM->hm.s.fNestedPaging)
13528 { /* likely */ }
13529 else
13530 {
13531#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13532 Assert(pVCpu->hm.s.fUsingDebugLoop);
13533#endif
13534 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13535 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13536 {
13537 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13538 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13539 }
13540 else
13541 {
13542 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13543 hmR0VmxSetPendingXcptDF(pVCpu);
13544 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13545 }
13546 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13547 return rc;
13548 }
13549
13550 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13551 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13552 if (pVmxTransient->fVectoringPF)
13553 {
13554 Assert(pVCpu->hm.s.Event.fPending);
13555 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13556 }
13557
13558 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13559 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13560 AssertRCReturn(rc, rc);
13561
13562 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13563 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13564
13565 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13566 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13567
13568 Log4Func(("#PF: rc=%Rrc\n", rc));
13569 if (rc == VINF_SUCCESS)
13570 {
13571 /*
13572 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13573 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13574 */
13575 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13576 TRPMResetTrap(pVCpu);
13577 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13578 return rc;
13579 }
13580
13581 if (rc == VINF_EM_RAW_GUEST_TRAP)
13582 {
13583 if (!pVmxTransient->fVectoringDoublePF)
13584 {
13585 /* It's a guest page fault and needs to be reflected to the guest. */
13586 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13587 TRPMResetTrap(pVCpu);
13588 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13589 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13590 uGstErrorCode, pVmxTransient->uExitQual);
13591 }
13592 else
13593 {
13594 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13595 TRPMResetTrap(pVCpu);
13596 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13597 hmR0VmxSetPendingXcptDF(pVCpu);
13598 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13599 }
13600
13601 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13602 return VINF_SUCCESS;
13603 }
13604
13605 TRPMResetTrap(pVCpu);
13606 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13607 return rc;
13608}
13609
13610/** @} */
13611
13612#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13613
13614/** @name Nested-guest VM-exit handlers.
13615 * @{
13616 */
13617/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13618/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13619/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13620
13621/**
13622 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13623 */
13624HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13625{
13626 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13627
13628 /** @todo NSTVMX: Vmclear. */
13629 hmR0VmxSetPendingXcptUD(pVCpu);
13630 return VINF_SUCCESS;
13631}
13632
13633
13634/**
13635 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13636 */
13637HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13638{
13639 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13640
13641 /** @todo NSTVMX: Vmlaunch. */
13642 hmR0VmxSetPendingXcptUD(pVCpu);
13643 return VINF_SUCCESS;
13644}
13645
13646
13647/**
13648 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13649 */
13650HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13651{
13652 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13653
13654 /** @todo NSTVMX: Vmptrld. */
13655 hmR0VmxSetPendingXcptUD(pVCpu);
13656 return VINF_SUCCESS;
13657}
13658
13659
13660/**
13661 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13662 */
13663HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13664{
13665 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13666
13667 /** @todo NSTVMX: Vmptrst. */
13668 hmR0VmxSetPendingXcptUD(pVCpu);
13669 return VINF_SUCCESS;
13670}
13671
13672
13673/**
13674 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13675 */
13676HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13677{
13678 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13679
13680 /** @todo NSTVMX: Vmread. */
13681 hmR0VmxSetPendingXcptUD(pVCpu);
13682 return VINF_SUCCESS;
13683}
13684
13685
13686/**
13687 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13688 */
13689HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13690{
13691 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13692
13693 /** @todo NSTVMX: Vmresume. */
13694 hmR0VmxSetPendingXcptUD(pVCpu);
13695 return VINF_SUCCESS;
13696}
13697
13698
13699/**
13700 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13701 */
13702HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13703{
13704 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13705
13706 /** @todo NSTVMX: Vmwrite. */
13707 hmR0VmxSetPendingXcptUD(pVCpu);
13708 return VINF_SUCCESS;
13709}
13710
13711
13712/**
13713 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13714 */
13715HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13716{
13717 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13718
13719 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13720 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13721 AssertRCReturn(rc, rc);
13722
13723 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13724 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13725 {
13726 /* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13727 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13728 }
13729 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13730 {
13731 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13732 rcStrict = VINF_SUCCESS;
13733 }
13734 return rcStrict;
13735}
13736
13737
13738/**
13739 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13740 */
13741HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13742{
13743 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13744
13745 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13746 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13747 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13748 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13749 AssertRCReturn(rc, rc);
13750
13751 VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToVmxInstr(pVCpu, pVmxTransient);
13752 if (rcStrict == VINF_SUCCESS)
13753 { /* likely */ }
13754 else if (rcStrict == VINF_HM_PENDING_XCPT)
13755 {
13756 Log4Func(("Privilege checks failed, raising xcpt %#x!\n", VMX_ENTRY_INT_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo)));
13757 return VINF_SUCCESS;
13758 }
13759 else
13760 {
13761 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13762 return rcStrict;
13763 }
13764
13765 RTGCPTR GCPtrVmxon;
13766 PCVMXEXITINSTRINFO pExitInstrInfo = &pVmxTransient->ExitInstrInfo;
13767 RTGCPTR const GCPtrDisp = pVmxTransient->uExitQual;
13768 rcStrict = hmR0VmxDecodeMemOperand(pVCpu, pExitInstrInfo, GCPtrDisp, false /*fIsWrite*/, &GCPtrVmxon);
13769 if (rcStrict == VINF_SUCCESS)
13770 { /* likely */ }
13771 else if (rcStrict == VINF_HM_PENDING_XCPT)
13772 {
13773 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", VMX_ENTRY_INT_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo)));
13774 return VINF_SUCCESS;
13775 }
13776 else
13777 {
13778 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13779 return rcStrict;
13780 }
13781
13782 rcStrict = IEMExecDecodedVmxon(pVCpu, pVmxTransient->cbInstr, GCPtrVmxon, pExitInstrInfo->u, GCPtrDisp);
13783 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13784 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13785 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13786 {
13787 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13788 rcStrict = VINF_SUCCESS;
13789 }
13790 return rcStrict;
13791}
13792
13793/** @} */
13794#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13795
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette