VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 85704

Last change on this file since 85704 was 85184, checked in by vboxsync, 4 years ago

VMM/IEMAll.cpp: Corrected to prototypes for inline functions. bugref:9794
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 651.0 KB
Line 
1/* $Id: IEMAll.cpp 85184 2020-07-10 13:22:05Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - All Contexts.
4 */
5
6/*
7 * Copyright (C) 2011-2020 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/** @page pg_iem IEM - Interpreted Execution Manager
20 *
21 * The interpreted exeuction manager (IEM) is for executing short guest code
22 * sequences that are causing too many exits / virtualization traps. It will
23 * also be used to interpret single instructions, thus replacing the selective
24 * interpreters in EM and IOM.
25 *
26 * Design goals:
27 * - Relatively small footprint, although we favour speed and correctness
28 * over size.
29 * - Reasonably fast.
30 * - Correctly handle lock prefixed instructions.
31 * - Complete instruction set - eventually.
32 * - Refactorable into a recompiler, maybe.
33 * - Replace EMInterpret*.
34 *
35 * Using the existing disassembler has been considered, however this is thought
36 * to conflict with speed as the disassembler chews things a bit too much while
37 * leaving us with a somewhat complicated state to interpret afterwards.
38 *
39 *
40 * The current code is very much work in progress. You've been warned!
41 *
42 *
43 * @section sec_iem_fpu_instr FPU Instructions
44 *
45 * On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
46 * same or equivalent instructions on the host FPU. To make life easy, we also
47 * let the FPU prioritize the unmasked exceptions for us. This however, only
48 * works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
49 * for FPU exception delivery, because with CR0.NE=0 there is a window where we
50 * can trigger spurious FPU exceptions.
51 *
52 * The guest FPU state is not loaded into the host CPU and kept there till we
53 * leave IEM because the calling conventions have declared an all year open
54 * season on much of the FPU state. For instance an innocent looking call to
55 * memcpy might end up using a whole bunch of XMM or MM registers if the
56 * particular implementation finds it worthwhile.
57 *
58 *
59 * @section sec_iem_logging Logging
60 *
61 * The IEM code uses the \"IEM\" log group for the main logging. The different
62 * logging levels/flags are generally used for the following purposes:
63 * - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
64 * - Flow (LogFlow): Basic enter/exit IEM state info.
65 * - Level 2 (Log2): ?
66 * - Level 3 (Log3): More detailed enter/exit IEM state info.
67 * - Level 4 (Log4): Decoding mnemonics w/ EIP.
68 * - Level 5 (Log5): Decoding details.
69 * - Level 6 (Log6): Enables/disables the lockstep comparison with REM.
70 * - Level 7 (Log7): iret++ execution logging.
71 * - Level 8 (Log8): Memory writes.
72 * - Level 9 (Log9): Memory reads.
73 *
74 */
75
76//#define IEM_LOG_MEMORY_WRITES
77#define IEM_IMPLEMENTS_TASKSWITCH
78
79/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */
80#ifdef _MSC_VER
81# pragma warning(disable:4505)
82#endif
83
84
85/*********************************************************************************************************************************
86* Header Files *
87*********************************************************************************************************************************/
88#define LOG_GROUP LOG_GROUP_IEM
89#define VMCPU_INCL_CPUM_GST_CTX
90#include <VBox/vmm/iem.h>
91#include <VBox/vmm/cpum.h>
92#include <VBox/vmm/apic.h>
93#include <VBox/vmm/pdm.h>
94#include <VBox/vmm/pgm.h>
95#include <VBox/vmm/iom.h>
96#include <VBox/vmm/em.h>
97#include <VBox/vmm/hm.h>
98#include <VBox/vmm/nem.h>
99#include <VBox/vmm/gim.h>
100#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
101# include <VBox/vmm/em.h>
102# include <VBox/vmm/hm_svm.h>
103#endif
104#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
105# include <VBox/vmm/hmvmxinline.h>
106#endif
107#include <VBox/vmm/tm.h>
108#include <VBox/vmm/dbgf.h>
109#include <VBox/vmm/dbgftrace.h>
110#include "IEMInternal.h"
111#include <VBox/vmm/vmcc.h>
112#include <VBox/log.h>
113#include <VBox/err.h>
114#include <VBox/param.h>
115#include <VBox/dis.h>
116#include <VBox/disopcode.h>
117#include <iprt/asm-math.h>
118#include <iprt/assert.h>
119#include <iprt/string.h>
120#include <iprt/x86.h>
121
122
123/*********************************************************************************************************************************
124* Structures and Typedefs *
125*********************************************************************************************************************************/
126/** @typedef PFNIEMOP
127 * Pointer to an opcode decoder function.
128 */
129
130/** @def FNIEMOP_DEF
131 * Define an opcode decoder function.
132 *
133 * We're using macors for this so that adding and removing parameters as well as
134 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
135 *
136 * @param a_Name The function name.
137 */
138
139/** @typedef PFNIEMOPRM
140 * Pointer to an opcode decoder function with RM byte.
141 */
142
143/** @def FNIEMOPRM_DEF
144 * Define an opcode decoder function with RM byte.
145 *
146 * We're using macors for this so that adding and removing parameters as well as
147 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
148 *
149 * @param a_Name The function name.
150 */
151
152#if defined(__GNUC__) && defined(RT_ARCH_X86)
153typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
154typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
155# define FNIEMOP_DEF(a_Name) \
156 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
157# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
158 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
159# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
160 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
161
162#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
163typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
164typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
165# define FNIEMOP_DEF(a_Name) \
166 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
167# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
168 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
169# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
170 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
171
172#elif defined(__GNUC__)
173typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
174typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
175# define FNIEMOP_DEF(a_Name) \
176 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
177# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
178 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
179# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
180 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
181
182#else
183typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
184typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
185# define FNIEMOP_DEF(a_Name) \
186 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
187# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
188 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
189# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
190 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
191
192#endif
193#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
194
195
196/**
197 * Selector descriptor table entry as fetched by iemMemFetchSelDesc.
198 */
199typedef union IEMSELDESC
200{
201 /** The legacy view. */
202 X86DESC Legacy;
203 /** The long mode view. */
204 X86DESC64 Long;
205} IEMSELDESC;
206/** Pointer to a selector descriptor table entry. */
207typedef IEMSELDESC *PIEMSELDESC;
208
209/**
210 * CPU exception classes.
211 */
212typedef enum IEMXCPTCLASS
213{
214 IEMXCPTCLASS_BENIGN,
215 IEMXCPTCLASS_CONTRIBUTORY,
216 IEMXCPTCLASS_PAGE_FAULT,
217 IEMXCPTCLASS_DOUBLE_FAULT
218} IEMXCPTCLASS;
219
220
221/*********************************************************************************************************************************
222* Defined Constants And Macros *
223*********************************************************************************************************************************/
224/** @def IEM_WITH_SETJMP
225 * Enables alternative status code handling using setjmps.
226 *
227 * This adds a bit of expense via the setjmp() call since it saves all the
228 * non-volatile registers. However, it eliminates return code checks and allows
229 * for more optimal return value passing (return regs instead of stack buffer).
230 */
231#if defined(DOXYGEN_RUNNING) || defined(RT_OS_WINDOWS) || 1
232# define IEM_WITH_SETJMP
233#endif
234
235/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
236 * due to GCC lacking knowledge about the value range of a switch. */
237#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
238
239/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
240#define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
241
242/**
243 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
244 * occation.
245 */
246#ifdef LOG_ENABLED
247# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
248 do { \
249 /*Log*/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
250 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
251 } while (0)
252#else
253# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
254 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
255#endif
256
257/**
258 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
259 * occation using the supplied logger statement.
260 *
261 * @param a_LoggerArgs What to log on failure.
262 */
263#ifdef LOG_ENABLED
264# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
265 do { \
266 LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
267 /*LogFunc(a_LoggerArgs);*/ \
268 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
269 } while (0)
270#else
271# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
272 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
273#endif
274
275/**
276 * Call an opcode decoder function.
277 *
278 * We're using macors for this so that adding and removing parameters can be
279 * done as we please. See FNIEMOP_DEF.
280 */
281#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
282
283/**
284 * Call a common opcode decoder function taking one extra argument.
285 *
286 * We're using macors for this so that adding and removing parameters can be
287 * done as we please. See FNIEMOP_DEF_1.
288 */
289#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
290
291/**
292 * Call a common opcode decoder function taking one extra argument.
293 *
294 * We're using macors for this so that adding and removing parameters can be
295 * done as we please. See FNIEMOP_DEF_1.
296 */
297#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
298
299/**
300 * Check if we're currently executing in real or virtual 8086 mode.
301 *
302 * @returns @c true if it is, @c false if not.
303 * @param a_pVCpu The IEM state of the current CPU.
304 */
305#define IEM_IS_REAL_OR_V86_MODE(a_pVCpu) (CPUMIsGuestInRealOrV86ModeEx(IEM_GET_CTX(a_pVCpu)))
306
307/**
308 * Check if we're currently executing in virtual 8086 mode.
309 *
310 * @returns @c true if it is, @c false if not.
311 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
312 */
313#define IEM_IS_V86_MODE(a_pVCpu) (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(a_pVCpu)))
314
315/**
316 * Check if we're currently executing in long mode.
317 *
318 * @returns @c true if it is, @c false if not.
319 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
320 */
321#define IEM_IS_LONG_MODE(a_pVCpu) (CPUMIsGuestInLongModeEx(IEM_GET_CTX(a_pVCpu)))
322
323/**
324 * Check if we're currently executing in a 64-bit code segment.
325 *
326 * @returns @c true if it is, @c false if not.
327 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
328 */
329#define IEM_IS_64BIT_CODE(a_pVCpu) (CPUMIsGuestIn64BitCodeEx(IEM_GET_CTX(a_pVCpu)))
330
331/**
332 * Check if we're currently executing in real mode.
333 *
334 * @returns @c true if it is, @c false if not.
335 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
336 */
337#define IEM_IS_REAL_MODE(a_pVCpu) (CPUMIsGuestInRealModeEx(IEM_GET_CTX(a_pVCpu)))
338
339/**
340 * Returns a (const) pointer to the CPUMFEATURES for the guest CPU.
341 * @returns PCCPUMFEATURES
342 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
343 */
344#define IEM_GET_GUEST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.GuestFeatures))
345
346/**
347 * Returns a (const) pointer to the CPUMFEATURES for the host CPU.
348 * @returns PCCPUMFEATURES
349 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
350 */
351#define IEM_GET_HOST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.HostFeatures))
352
353/**
354 * Evaluates to true if we're presenting an Intel CPU to the guest.
355 */
356#define IEM_IS_GUEST_CPU_INTEL(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL )
357
358/**
359 * Evaluates to true if we're presenting an AMD CPU to the guest.
360 */
361#define IEM_IS_GUEST_CPU_AMD(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_AMD || (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_HYGON )
362
363/**
364 * Check if the address is canonical.
365 */
366#define IEM_IS_CANONICAL(a_u64Addr) X86_IS_CANONICAL(a_u64Addr)
367
368/**
369 * Gets the effective VEX.VVVV value.
370 *
371 * The 4th bit is ignored if not 64-bit code.
372 * @returns effective V-register value.
373 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
374 */
375#define IEM_GET_EFFECTIVE_VVVV(a_pVCpu) \
376 ((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT ? (a_pVCpu)->iem.s.uVex3rdReg : (a_pVCpu)->iem.s.uVex3rdReg & 7)
377
378/** @def IEM_USE_UNALIGNED_DATA_ACCESS
379 * Use unaligned accesses instead of elaborate byte assembly. */
380#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) || defined(DOXYGEN_RUNNING)
381# define IEM_USE_UNALIGNED_DATA_ACCESS
382#endif
383
384#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
385
386/**
387 * Check if the guest has entered VMX root operation.
388 */
389# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxRootMode(IEM_GET_CTX(a_pVCpu)))
390
391/**
392 * Check if the guest has entered VMX non-root operation.
393 */
394# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(a_pVCpu)))
395
396/**
397 * Check if the nested-guest has the given Pin-based VM-execution control set.
398 */
399# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_PinCtl) \
400 (CPUMIsGuestVmxPinCtlsSet(IEM_GET_CTX(a_pVCpu), (a_PinCtl)))
401
402/**
403 * Check if the nested-guest has the given Processor-based VM-execution control set.
404 */
405#define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_ProcCtl) \
406 (CPUMIsGuestVmxProcCtlsSet(IEM_GET_CTX(a_pVCpu), (a_ProcCtl)))
407
408/**
409 * Check if the nested-guest has the given Secondary Processor-based VM-execution
410 * control set.
411 */
412#define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_ProcCtl2) \
413 (CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(a_pVCpu), (a_ProcCtl2)))
414
415/**
416 * Invokes the VMX VM-exit handler for an instruction intercept.
417 */
418# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) \
419 do { return iemVmxVmexitInstr((a_pVCpu), (a_uExitReason), (a_cbInstr)); } while (0)
420
421/**
422 * Invokes the VMX VM-exit handler for an instruction intercept where the
423 * instruction provides additional VM-exit information.
424 */
425# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) \
426 do { return iemVmxVmexitInstrNeedsInfo((a_pVCpu), (a_uExitReason), (a_uInstrId), (a_cbInstr)); } while (0)
427
428/**
429 * Invokes the VMX VM-exit handler for a task switch.
430 */
431# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) \
432 do { return iemVmxVmexitTaskSwitch((a_pVCpu), (a_enmTaskSwitch), (a_SelNewTss), (a_cbInstr)); } while (0)
433
434/**
435 * Invokes the VMX VM-exit handler for MWAIT.
436 */
437# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) \
438 do { return iemVmxVmexitInstrMwait((a_pVCpu), (a_fMonitorArmed), (a_cbInstr)); } while (0)
439
440/**
441 * Invokes the VMX VM-exit handler.
442 */
443# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) \
444 do { return iemVmxVmexit((a_pVCpu), (a_uExitReason), (a_uExitQual)); } while (0)
445
446#else
447# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (false)
448# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (false)
449# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_cbInstr) (false)
450# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_cbInstr) (false)
451# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_cbInstr) (false)
452# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
453# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
454# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
455# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
456# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) do { return VERR_VMX_IPE_1; } while (0)
457
458#endif
459
460#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
461/**
462 * Check if an SVM control/instruction intercept is set.
463 */
464# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) \
465 (CPUMIsGuestSvmCtrlInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_Intercept)))
466
467/**
468 * Check if an SVM read CRx intercept is set.
469 */
470# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
471 (CPUMIsGuestSvmReadCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
472
473/**
474 * Check if an SVM write CRx intercept is set.
475 */
476# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
477 (CPUMIsGuestSvmWriteCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
478
479/**
480 * Check if an SVM read DRx intercept is set.
481 */
482# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
483 (CPUMIsGuestSvmReadDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
484
485/**
486 * Check if an SVM write DRx intercept is set.
487 */
488# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
489 (CPUMIsGuestSvmWriteDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
490
491/**
492 * Check if an SVM exception intercept is set.
493 */
494# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) \
495 (CPUMIsGuestSvmXcptInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uVector)))
496
497/**
498 * Invokes the SVM \#VMEXIT handler for the nested-guest.
499 */
500# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
501 do { return iemSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); } while (0)
502
503/**
504 * Invokes the 'MOV CRx' SVM \#VMEXIT handler after constructing the
505 * corresponding decode assist information.
506 */
507# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) \
508 do \
509 { \
510 uint64_t uExitInfo1; \
511 if ( IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmDecodeAssists \
512 && (a_enmAccessCrX) == IEMACCESSCRX_MOV_CRX) \
513 uExitInfo1 = SVM_EXIT1_MOV_CRX_MASK | ((a_iGReg) & 7); \
514 else \
515 uExitInfo1 = 0; \
516 IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, uExitInfo1, 0); \
517 } while (0)
518
519/** Check and handles SVM nested-guest instruction intercept and updates
520 * NRIP if needed.
521 */
522# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
523 do \
524 { \
525 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept)) \
526 { \
527 IEM_SVM_UPDATE_NRIP(a_pVCpu); \
528 IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2); \
529 } \
530 } while (0)
531
532/** Checks and handles SVM nested-guest CR0 read intercept. */
533# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) \
534 do \
535 { \
536 if (!IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, 0)) \
537 { /* probably likely */ } \
538 else \
539 { \
540 IEM_SVM_UPDATE_NRIP(a_pVCpu); \
541 IEM_SVM_VMEXIT_RET(a_pVCpu, SVM_EXIT_READ_CR0, a_uExitInfo1, a_uExitInfo2); \
542 } \
543 } while (0)
544
545/**
546 * Updates the NextRIP (NRI) field in the nested-guest VMCB.
547 */
548# define IEM_SVM_UPDATE_NRIP(a_pVCpu) \
549 do { \
550 if (IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmNextRipSave) \
551 CPUMGuestSvmUpdateNRip(a_pVCpu, IEM_GET_CTX(a_pVCpu), IEM_GET_INSTR_LEN(a_pVCpu)); \
552 } while (0)
553
554#else
555# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) (false)
556# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
557# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
558# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
559# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
560# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) (false)
561# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { return VERR_SVM_IPE_1; } while (0)
562# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) do { return VERR_SVM_IPE_1; } while (0)
563# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { } while (0)
564# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) do { } while (0)
565# define IEM_SVM_UPDATE_NRIP(a_pVCpu) do { } while (0)
566
567#endif
568
569
570/*********************************************************************************************************************************
571* Global Variables *
572*********************************************************************************************************************************/
573extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
574
575
576/** Function table for the ADD instruction. */
577IEM_STATIC const IEMOPBINSIZES g_iemAImpl_add =
578{
579 iemAImpl_add_u8, iemAImpl_add_u8_locked,
580 iemAImpl_add_u16, iemAImpl_add_u16_locked,
581 iemAImpl_add_u32, iemAImpl_add_u32_locked,
582 iemAImpl_add_u64, iemAImpl_add_u64_locked
583};
584
585/** Function table for the ADC instruction. */
586IEM_STATIC const IEMOPBINSIZES g_iemAImpl_adc =
587{
588 iemAImpl_adc_u8, iemAImpl_adc_u8_locked,
589 iemAImpl_adc_u16, iemAImpl_adc_u16_locked,
590 iemAImpl_adc_u32, iemAImpl_adc_u32_locked,
591 iemAImpl_adc_u64, iemAImpl_adc_u64_locked
592};
593
594/** Function table for the SUB instruction. */
595IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sub =
596{
597 iemAImpl_sub_u8, iemAImpl_sub_u8_locked,
598 iemAImpl_sub_u16, iemAImpl_sub_u16_locked,
599 iemAImpl_sub_u32, iemAImpl_sub_u32_locked,
600 iemAImpl_sub_u64, iemAImpl_sub_u64_locked
601};
602
603/** Function table for the SBB instruction. */
604IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sbb =
605{
606 iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked,
607 iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked,
608 iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked,
609 iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked
610};
611
612/** Function table for the OR instruction. */
613IEM_STATIC const IEMOPBINSIZES g_iemAImpl_or =
614{
615 iemAImpl_or_u8, iemAImpl_or_u8_locked,
616 iemAImpl_or_u16, iemAImpl_or_u16_locked,
617 iemAImpl_or_u32, iemAImpl_or_u32_locked,
618 iemAImpl_or_u64, iemAImpl_or_u64_locked
619};
620
621/** Function table for the XOR instruction. */
622IEM_STATIC const IEMOPBINSIZES g_iemAImpl_xor =
623{
624 iemAImpl_xor_u8, iemAImpl_xor_u8_locked,
625 iemAImpl_xor_u16, iemAImpl_xor_u16_locked,
626 iemAImpl_xor_u32, iemAImpl_xor_u32_locked,
627 iemAImpl_xor_u64, iemAImpl_xor_u64_locked
628};
629
630/** Function table for the AND instruction. */
631IEM_STATIC const IEMOPBINSIZES g_iemAImpl_and =
632{
633 iemAImpl_and_u8, iemAImpl_and_u8_locked,
634 iemAImpl_and_u16, iemAImpl_and_u16_locked,
635 iemAImpl_and_u32, iemAImpl_and_u32_locked,
636 iemAImpl_and_u64, iemAImpl_and_u64_locked
637};
638
639/** Function table for the CMP instruction.
640 * @remarks Making operand order ASSUMPTIONS.
641 */
642IEM_STATIC const IEMOPBINSIZES g_iemAImpl_cmp =
643{
644 iemAImpl_cmp_u8, NULL,
645 iemAImpl_cmp_u16, NULL,
646 iemAImpl_cmp_u32, NULL,
647 iemAImpl_cmp_u64, NULL
648};
649
650/** Function table for the TEST instruction.
651 * @remarks Making operand order ASSUMPTIONS.
652 */
653IEM_STATIC const IEMOPBINSIZES g_iemAImpl_test =
654{
655 iemAImpl_test_u8, NULL,
656 iemAImpl_test_u16, NULL,
657 iemAImpl_test_u32, NULL,
658 iemAImpl_test_u64, NULL
659};
660
661/** Function table for the BT instruction. */
662IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bt =
663{
664 NULL, NULL,
665 iemAImpl_bt_u16, NULL,
666 iemAImpl_bt_u32, NULL,
667 iemAImpl_bt_u64, NULL
668};
669
670/** Function table for the BTC instruction. */
671IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btc =
672{
673 NULL, NULL,
674 iemAImpl_btc_u16, iemAImpl_btc_u16_locked,
675 iemAImpl_btc_u32, iemAImpl_btc_u32_locked,
676 iemAImpl_btc_u64, iemAImpl_btc_u64_locked
677};
678
679/** Function table for the BTR instruction. */
680IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btr =
681{
682 NULL, NULL,
683 iemAImpl_btr_u16, iemAImpl_btr_u16_locked,
684 iemAImpl_btr_u32, iemAImpl_btr_u32_locked,
685 iemAImpl_btr_u64, iemAImpl_btr_u64_locked
686};
687
688/** Function table for the BTS instruction. */
689IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bts =
690{
691 NULL, NULL,
692 iemAImpl_bts_u16, iemAImpl_bts_u16_locked,
693 iemAImpl_bts_u32, iemAImpl_bts_u32_locked,
694 iemAImpl_bts_u64, iemAImpl_bts_u64_locked
695};
696
697/** Function table for the BSF instruction. */
698IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsf =
699{
700 NULL, NULL,
701 iemAImpl_bsf_u16, NULL,
702 iemAImpl_bsf_u32, NULL,
703 iemAImpl_bsf_u64, NULL
704};
705
706/** Function table for the BSR instruction. */
707IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsr =
708{
709 NULL, NULL,
710 iemAImpl_bsr_u16, NULL,
711 iemAImpl_bsr_u32, NULL,
712 iemAImpl_bsr_u64, NULL
713};
714
715/** Function table for the IMUL instruction. */
716IEM_STATIC const IEMOPBINSIZES g_iemAImpl_imul_two =
717{
718 NULL, NULL,
719 iemAImpl_imul_two_u16, NULL,
720 iemAImpl_imul_two_u32, NULL,
721 iemAImpl_imul_two_u64, NULL
722};
723
724/** Group 1 /r lookup table. */
725IEM_STATIC const PCIEMOPBINSIZES g_apIemImplGrp1[8] =
726{
727 &g_iemAImpl_add,
728 &g_iemAImpl_or,
729 &g_iemAImpl_adc,
730 &g_iemAImpl_sbb,
731 &g_iemAImpl_and,
732 &g_iemAImpl_sub,
733 &g_iemAImpl_xor,
734 &g_iemAImpl_cmp
735};
736
737/** Function table for the INC instruction. */
738IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_inc =
739{
740 iemAImpl_inc_u8, iemAImpl_inc_u8_locked,
741 iemAImpl_inc_u16, iemAImpl_inc_u16_locked,
742 iemAImpl_inc_u32, iemAImpl_inc_u32_locked,
743 iemAImpl_inc_u64, iemAImpl_inc_u64_locked
744};
745
746/** Function table for the DEC instruction. */
747IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_dec =
748{
749 iemAImpl_dec_u8, iemAImpl_dec_u8_locked,
750 iemAImpl_dec_u16, iemAImpl_dec_u16_locked,
751 iemAImpl_dec_u32, iemAImpl_dec_u32_locked,
752 iemAImpl_dec_u64, iemAImpl_dec_u64_locked
753};
754
755/** Function table for the NEG instruction. */
756IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_neg =
757{
758 iemAImpl_neg_u8, iemAImpl_neg_u8_locked,
759 iemAImpl_neg_u16, iemAImpl_neg_u16_locked,
760 iemAImpl_neg_u32, iemAImpl_neg_u32_locked,
761 iemAImpl_neg_u64, iemAImpl_neg_u64_locked
762};
763
764/** Function table for the NOT instruction. */
765IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_not =
766{
767 iemAImpl_not_u8, iemAImpl_not_u8_locked,
768 iemAImpl_not_u16, iemAImpl_not_u16_locked,
769 iemAImpl_not_u32, iemAImpl_not_u32_locked,
770 iemAImpl_not_u64, iemAImpl_not_u64_locked
771};
772
773
774/** Function table for the ROL instruction. */
775IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rol =
776{
777 iemAImpl_rol_u8,
778 iemAImpl_rol_u16,
779 iemAImpl_rol_u32,
780 iemAImpl_rol_u64
781};
782
783/** Function table for the ROR instruction. */
784IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_ror =
785{
786 iemAImpl_ror_u8,
787 iemAImpl_ror_u16,
788 iemAImpl_ror_u32,
789 iemAImpl_ror_u64
790};
791
792/** Function table for the RCL instruction. */
793IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcl =
794{
795 iemAImpl_rcl_u8,
796 iemAImpl_rcl_u16,
797 iemAImpl_rcl_u32,
798 iemAImpl_rcl_u64
799};
800
801/** Function table for the RCR instruction. */
802IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcr =
803{
804 iemAImpl_rcr_u8,
805 iemAImpl_rcr_u16,
806 iemAImpl_rcr_u32,
807 iemAImpl_rcr_u64
808};
809
810/** Function table for the SHL instruction. */
811IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shl =
812{
813 iemAImpl_shl_u8,
814 iemAImpl_shl_u16,
815 iemAImpl_shl_u32,
816 iemAImpl_shl_u64
817};
818
819/** Function table for the SHR instruction. */
820IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shr =
821{
822 iemAImpl_shr_u8,
823 iemAImpl_shr_u16,
824 iemAImpl_shr_u32,
825 iemAImpl_shr_u64
826};
827
828/** Function table for the SAR instruction. */
829IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_sar =
830{
831 iemAImpl_sar_u8,
832 iemAImpl_sar_u16,
833 iemAImpl_sar_u32,
834 iemAImpl_sar_u64
835};
836
837
838/** Function table for the MUL instruction. */
839IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_mul =
840{
841 iemAImpl_mul_u8,
842 iemAImpl_mul_u16,
843 iemAImpl_mul_u32,
844 iemAImpl_mul_u64
845};
846
847/** Function table for the IMUL instruction working implicitly on rAX. */
848IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_imul =
849{
850 iemAImpl_imul_u8,
851 iemAImpl_imul_u16,
852 iemAImpl_imul_u32,
853 iemAImpl_imul_u64
854};
855
856/** Function table for the DIV instruction. */
857IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_div =
858{
859 iemAImpl_div_u8,
860 iemAImpl_div_u16,
861 iemAImpl_div_u32,
862 iemAImpl_div_u64
863};
864
865/** Function table for the MUL instruction. */
866IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_idiv =
867{
868 iemAImpl_idiv_u8,
869 iemAImpl_idiv_u16,
870 iemAImpl_idiv_u32,
871 iemAImpl_idiv_u64
872};
873
874/** Function table for the SHLD instruction */
875IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shld =
876{
877 iemAImpl_shld_u16,
878 iemAImpl_shld_u32,
879 iemAImpl_shld_u64,
880};
881
882/** Function table for the SHRD instruction */
883IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd =
884{
885 iemAImpl_shrd_u16,
886 iemAImpl_shrd_u32,
887 iemAImpl_shrd_u64,
888};
889
890
891/** Function table for the PUNPCKLBW instruction */
892IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklbw = { iemAImpl_punpcklbw_u64, iemAImpl_punpcklbw_u128 };
893/** Function table for the PUNPCKLBD instruction */
894IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklwd = { iemAImpl_punpcklwd_u64, iemAImpl_punpcklwd_u128 };
895/** Function table for the PUNPCKLDQ instruction */
896IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpckldq = { iemAImpl_punpckldq_u64, iemAImpl_punpckldq_u128 };
897/** Function table for the PUNPCKLQDQ instruction */
898IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklqdq = { NULL, iemAImpl_punpcklqdq_u128 };
899
900/** Function table for the PUNPCKHBW instruction */
901IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhbw = { iemAImpl_punpckhbw_u64, iemAImpl_punpckhbw_u128 };
902/** Function table for the PUNPCKHBD instruction */
903IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhwd = { iemAImpl_punpckhwd_u64, iemAImpl_punpckhwd_u128 };
904/** Function table for the PUNPCKHDQ instruction */
905IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhdq = { iemAImpl_punpckhdq_u64, iemAImpl_punpckhdq_u128 };
906/** Function table for the PUNPCKHQDQ instruction */
907IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhqdq = { NULL, iemAImpl_punpckhqdq_u128 };
908
909/** Function table for the PXOR instruction */
910IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pxor = { iemAImpl_pxor_u64, iemAImpl_pxor_u128 };
911/** Function table for the PCMPEQB instruction */
912IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqb = { iemAImpl_pcmpeqb_u64, iemAImpl_pcmpeqb_u128 };
913/** Function table for the PCMPEQW instruction */
914IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqw = { iemAImpl_pcmpeqw_u64, iemAImpl_pcmpeqw_u128 };
915/** Function table for the PCMPEQD instruction */
916IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqd = { iemAImpl_pcmpeqd_u64, iemAImpl_pcmpeqd_u128 };
917
918
919#if defined(IEM_LOG_MEMORY_WRITES)
920/** What IEM just wrote. */
921uint8_t g_abIemWrote[256];
922/** How much IEM just wrote. */
923size_t g_cbIemWrote;
924#endif
925
926
927/*********************************************************************************************************************************
928* Internal Functions *
929*********************************************************************************************************************************/
930IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr);
931IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu);
932IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu);
933IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel);
934/*IEM_STATIC VBOXSTRICTRC iemRaiseSelectorNotPresent(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);*/
935IEM_STATIC VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel);
936IEM_STATIC VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr);
937IEM_STATIC VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel);
938IEM_STATIC VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr);
939IEM_STATIC VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr);
940IEM_STATIC VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu);
941IEM_STATIC VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL uSel);
942IEM_STATIC VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
943IEM_STATIC VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel);
944IEM_STATIC VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
945IEM_STATIC VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
946IEM_STATIC VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu);
947#ifdef IEM_WITH_SETJMP
948DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
949DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu);
950DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
951DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel);
952DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
953#endif
954
955IEM_STATIC VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess);
956IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess);
957IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
958IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
959IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
960IEM_STATIC VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
961IEM_STATIC VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
962IEM_STATIC VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
963IEM_STATIC VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
964IEM_STATIC VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt, uint16_t uErrorCode);
965IEM_STATIC VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt);
966IEM_STATIC VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp);
967IEM_STATIC VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void **ppvMem, uint64_t *puNewRsp);
968IEM_STATIC VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value);
969IEM_STATIC VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value);
970IEM_STATIC VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel);
971DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg);
972DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg);
973
974#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
975IEM_STATIC VBOXSTRICTRC iemVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual);
976IEM_STATIC VBOXSTRICTRC iemVmxVmexitTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, RTSEL SelNewTss, uint8_t cbInstr);
977IEM_STATIC VBOXSTRICTRC iemVmxVmexitEvent(PVMCPUCC pVCpu, uint8_t uVector, uint32_t fFlags, uint32_t uErrCode, uint64_t uCr2, uint8_t cbInstr);
978IEM_STATIC VBOXSTRICTRC iemVmxVmexitEventDoubleFault(PVMCPUCC pVCpu);
979IEM_STATIC VBOXSTRICTRC iemVmxVirtApicAccessMem(PVMCPUCC pVCpu, uint16_t offAccess, size_t cbAccess, void *pvData, uint32_t fAccess);
980IEM_STATIC VBOXSTRICTRC iemVmxVirtApicAccessMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value);
981IEM_STATIC VBOXSTRICTRC iemVmxVirtApicAccessMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Value);
982#endif
983
984#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
985IEM_STATIC VBOXSTRICTRC iemSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2);
986IEM_STATIC VBOXSTRICTRC iemHandleSvmEventIntercept(PVMCPUCC pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2);
987#endif
988
989
990/**
991 * Sets the pass up status.
992 *
993 * @returns VINF_SUCCESS.
994 * @param pVCpu The cross context virtual CPU structure of the
995 * calling thread.
996 * @param rcPassUp The pass up status. Must be informational.
997 * VINF_SUCCESS is not allowed.
998 */
999IEM_STATIC int iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp)
1000{
1001 AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
1002
1003 int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
1004 if (rcOldPassUp == VINF_SUCCESS)
1005 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
1006 /* If both are EM scheduling codes, use EM priority rules. */
1007 else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
1008 && rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
1009 {
1010 if (rcPassUp < rcOldPassUp)
1011 {
1012 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
1013 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
1014 }
1015 else
1016 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
1017 }
1018 /* Override EM scheduling with specific status code. */
1019 else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
1020 {
1021 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
1022 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
1023 }
1024 /* Don't override specific status code, first come first served. */
1025 else
1026 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
1027 return VINF_SUCCESS;
1028}
1029
1030
1031/**
1032 * Calculates the CPU mode.
1033 *
1034 * This is mainly for updating IEMCPU::enmCpuMode.
1035 *
1036 * @returns CPU mode.
1037 * @param pVCpu The cross context virtual CPU structure of the
1038 * calling thread.
1039 */
1040DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu)
1041{
1042 if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
1043 return IEMMODE_64BIT;
1044 if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
1045 return IEMMODE_32BIT;
1046 return IEMMODE_16BIT;
1047}
1048
1049
1050/**
1051 * Initializes the execution state.
1052 *
1053 * @param pVCpu The cross context virtual CPU structure of the
1054 * calling thread.
1055 * @param fBypassHandlers Whether to bypass access handlers.
1056 *
1057 * @remarks Callers of this must call iemUninitExec() to undo potentially fatal
1058 * side-effects in strict builds.
1059 */
1060DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers)
1061{
1062 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
1063 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
1064 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
1065 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1066 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
1067 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
1068 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
1069 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
1070 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
1071 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
1072
1073 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
1074 pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
1075#ifdef VBOX_STRICT
1076 pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
1077 pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
1078 pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
1079 pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
1080 pVCpu->iem.s.fPrefixes = 0xfeedbeef;
1081 pVCpu->iem.s.uRexReg = 127;
1082 pVCpu->iem.s.uRexB = 127;
1083 pVCpu->iem.s.offModRm = 127;
1084 pVCpu->iem.s.uRexIndex = 127;
1085 pVCpu->iem.s.iEffSeg = 127;
1086 pVCpu->iem.s.idxPrefix = 127;
1087 pVCpu->iem.s.uVex3rdReg = 127;
1088 pVCpu->iem.s.uVexLength = 127;
1089 pVCpu->iem.s.fEvexStuff = 127;
1090 pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
1091# ifdef IEM_WITH_CODE_TLB
1092 pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
1093 pVCpu->iem.s.pbInstrBuf = NULL;
1094 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
1095 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
1096 pVCpu->iem.s.offCurInstrStart = INT16_MAX;
1097 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
1098# else
1099 pVCpu->iem.s.offOpcode = 127;
1100 pVCpu->iem.s.cbOpcode = 127;
1101# endif
1102#endif
1103
1104 pVCpu->iem.s.cActiveMappings = 0;
1105 pVCpu->iem.s.iNextMapping = 0;
1106 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
1107 pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
1108#if 0
1109#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1110 if ( CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
1111 && CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
1112 {
1113 PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1114 Assert(pVmcs);
1115 RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
1116 if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
1117 {
1118 int rc = PGMHandlerPhysicalRegister(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess, GCPhysApicAccess + X86_PAGE_4K_SIZE - 1,
1119 pVCpu->iem.s.hVmxApicAccessPage, NIL_RTR3PTR /* pvUserR3 */,
1120 NIL_RTR0PTR /* pvUserR0 */, NIL_RTRCPTR /* pvUserRC */, NULL /* pszDesc */);
1121 AssertRC(rc);
1122 }
1123 }
1124#endif
1125#endif
1126}
1127
1128#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
1129/**
1130 * Performs a minimal reinitialization of the execution state.
1131 *
1132 * This is intended to be used by VM-exits, SMM, LOADALL and other similar
1133 * 'world-switch' types operations on the CPU. Currently only nested
1134 * hardware-virtualization uses it.
1135 *
1136 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1137 */
1138IEM_STATIC void iemReInitExec(PVMCPUCC pVCpu)
1139{
1140 IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
1141 uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
1142
1143 pVCpu->iem.s.uCpl = uCpl;
1144 pVCpu->iem.s.enmCpuMode = enmMode;
1145 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
1146 pVCpu->iem.s.enmEffAddrMode = enmMode;
1147 if (enmMode != IEMMODE_64BIT)
1148 {
1149 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
1150 pVCpu->iem.s.enmEffOpSize = enmMode;
1151 }
1152 else
1153 {
1154 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
1155 pVCpu->iem.s.enmEffOpSize = enmMode;
1156 }
1157 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
1158#ifndef IEM_WITH_CODE_TLB
1159 /** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
1160 pVCpu->iem.s.offOpcode = 0;
1161 pVCpu->iem.s.cbOpcode = 0;
1162#endif
1163 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
1164}
1165#endif
1166
1167/**
1168 * Counterpart to #iemInitExec that undoes evil strict-build stuff.
1169 *
1170 * @param pVCpu The cross context virtual CPU structure of the
1171 * calling thread.
1172 */
1173DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu)
1174{
1175 /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
1176#ifdef VBOX_STRICT
1177# ifdef IEM_WITH_CODE_TLB
1178 NOREF(pVCpu);
1179# else
1180 pVCpu->iem.s.cbOpcode = 0;
1181# endif
1182#else
1183 NOREF(pVCpu);
1184#endif
1185}
1186
1187
1188/**
1189 * Initializes the decoder state.
1190 *
1191 * iemReInitDecoder is mostly a copy of this function.
1192 *
1193 * @param pVCpu The cross context virtual CPU structure of the
1194 * calling thread.
1195 * @param fBypassHandlers Whether to bypass access handlers.
1196 */
1197DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, bool fBypassHandlers)
1198{
1199 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
1200 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
1201 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
1202 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1203 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
1204 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
1205 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
1206 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
1207 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
1208 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
1209
1210 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
1211 IEMMODE enmMode = iemCalcCpuMode(pVCpu);
1212 pVCpu->iem.s.enmCpuMode = enmMode;
1213 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
1214 pVCpu->iem.s.enmEffAddrMode = enmMode;
1215 if (enmMode != IEMMODE_64BIT)
1216 {
1217 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
1218 pVCpu->iem.s.enmEffOpSize = enmMode;
1219 }
1220 else
1221 {
1222 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
1223 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
1224 }
1225 pVCpu->iem.s.fPrefixes = 0;
1226 pVCpu->iem.s.uRexReg = 0;
1227 pVCpu->iem.s.uRexB = 0;
1228 pVCpu->iem.s.uRexIndex = 0;
1229 pVCpu->iem.s.idxPrefix = 0;
1230 pVCpu->iem.s.uVex3rdReg = 0;
1231 pVCpu->iem.s.uVexLength = 0;
1232 pVCpu->iem.s.fEvexStuff = 0;
1233 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
1234#ifdef IEM_WITH_CODE_TLB
1235 pVCpu->iem.s.pbInstrBuf = NULL;
1236 pVCpu->iem.s.offInstrNextByte = 0;
1237 pVCpu->iem.s.offCurInstrStart = 0;
1238# ifdef VBOX_STRICT
1239 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
1240 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
1241 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
1242# endif
1243#else
1244 pVCpu->iem.s.offOpcode = 0;
1245 pVCpu->iem.s.cbOpcode = 0;
1246#endif
1247 pVCpu->iem.s.offModRm = 0;
1248 pVCpu->iem.s.cActiveMappings = 0;
1249 pVCpu->iem.s.iNextMapping = 0;
1250 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
1251 pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
1252
1253#ifdef DBGFTRACE_ENABLED
1254 switch (enmMode)
1255 {
1256 case IEMMODE_64BIT:
1257 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
1258 break;
1259 case IEMMODE_32BIT:
1260 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1261 break;
1262 case IEMMODE_16BIT:
1263 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1264 break;
1265 }
1266#endif
1267}
1268
1269
1270/**
1271 * Reinitializes the decoder state 2nd+ loop of IEMExecLots.
1272 *
1273 * This is mostly a copy of iemInitDecoder.
1274 *
1275 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1276 */
1277DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu)
1278{
1279 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
1280 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
1281 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1282 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
1283 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
1284 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
1285 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
1286 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
1287 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
1288
1289 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu); /** @todo this should be updated during execution! */
1290 IEMMODE enmMode = iemCalcCpuMode(pVCpu);
1291 pVCpu->iem.s.enmCpuMode = enmMode; /** @todo this should be updated during execution! */
1292 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
1293 pVCpu->iem.s.enmEffAddrMode = enmMode;
1294 if (enmMode != IEMMODE_64BIT)
1295 {
1296 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
1297 pVCpu->iem.s.enmEffOpSize = enmMode;
1298 }
1299 else
1300 {
1301 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
1302 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
1303 }
1304 pVCpu->iem.s.fPrefixes = 0;
1305 pVCpu->iem.s.uRexReg = 0;
1306 pVCpu->iem.s.uRexB = 0;
1307 pVCpu->iem.s.uRexIndex = 0;
1308 pVCpu->iem.s.idxPrefix = 0;
1309 pVCpu->iem.s.uVex3rdReg = 0;
1310 pVCpu->iem.s.uVexLength = 0;
1311 pVCpu->iem.s.fEvexStuff = 0;
1312 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
1313#ifdef IEM_WITH_CODE_TLB
1314 if (pVCpu->iem.s.pbInstrBuf)
1315 {
1316 uint64_t off = (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rip : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)
1317 - pVCpu->iem.s.uInstrBufPc;
1318 if (off < pVCpu->iem.s.cbInstrBufTotal)
1319 {
1320 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
1321 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
1322 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
1323 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
1324 else
1325 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
1326 }
1327 else
1328 {
1329 pVCpu->iem.s.pbInstrBuf = NULL;
1330 pVCpu->iem.s.offInstrNextByte = 0;
1331 pVCpu->iem.s.offCurInstrStart = 0;
1332 pVCpu->iem.s.cbInstrBuf = 0;
1333 pVCpu->iem.s.cbInstrBufTotal = 0;
1334 }
1335 }
1336 else
1337 {
1338 pVCpu->iem.s.offInstrNextByte = 0;
1339 pVCpu->iem.s.offCurInstrStart = 0;
1340 pVCpu->iem.s.cbInstrBuf = 0;
1341 pVCpu->iem.s.cbInstrBufTotal = 0;
1342 }
1343#else
1344 pVCpu->iem.s.cbOpcode = 0;
1345 pVCpu->iem.s.offOpcode = 0;
1346#endif
1347 pVCpu->iem.s.offModRm = 0;
1348 Assert(pVCpu->iem.s.cActiveMappings == 0);
1349 pVCpu->iem.s.iNextMapping = 0;
1350 Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
1351 Assert(pVCpu->iem.s.fBypassHandlers == false);
1352
1353#ifdef DBGFTRACE_ENABLED
1354 switch (enmMode)
1355 {
1356 case IEMMODE_64BIT:
1357 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
1358 break;
1359 case IEMMODE_32BIT:
1360 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1361 break;
1362 case IEMMODE_16BIT:
1363 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1364 break;
1365 }
1366#endif
1367}
1368
1369
1370
1371/**
1372 * Prefetch opcodes the first time when starting executing.
1373 *
1374 * @returns Strict VBox status code.
1375 * @param pVCpu The cross context virtual CPU structure of the
1376 * calling thread.
1377 * @param fBypassHandlers Whether to bypass access handlers.
1378 */
1379IEM_STATIC VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, bool fBypassHandlers)
1380{
1381 iemInitDecoder(pVCpu, fBypassHandlers);
1382
1383#ifdef IEM_WITH_CODE_TLB
1384 /** @todo Do ITLB lookup here. */
1385
1386#else /* !IEM_WITH_CODE_TLB */
1387
1388 /*
1389 * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
1390 *
1391 * First translate CS:rIP to a physical address.
1392 */
1393 uint32_t cbToTryRead;
1394 RTGCPTR GCPtrPC;
1395 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1396 {
1397 cbToTryRead = PAGE_SIZE;
1398 GCPtrPC = pVCpu->cpum.GstCtx.rip;
1399 if (IEM_IS_CANONICAL(GCPtrPC))
1400 cbToTryRead = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
1401 else
1402 return iemRaiseGeneralProtectionFault0(pVCpu);
1403 }
1404 else
1405 {
1406 uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
1407 AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
1408 if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
1409 cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
1410 else
1411 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1412 if (cbToTryRead) { /* likely */ }
1413 else /* overflowed */
1414 {
1415 Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1416 cbToTryRead = UINT32_MAX;
1417 }
1418 GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
1419 Assert(GCPtrPC <= UINT32_MAX);
1420 }
1421
1422 RTGCPHYS GCPhys;
1423 uint64_t fFlags;
1424 int rc = PGMGstGetPage(pVCpu, GCPtrPC, &fFlags, &GCPhys);
1425 if (RT_SUCCESS(rc)) { /* probable */ }
1426 else
1427 {
1428 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
1429 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
1430 }
1431 if ((fFlags & X86_PTE_US) || pVCpu->iem.s.uCpl != 3) { /* likely */ }
1432 else
1433 {
1434 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
1435 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1436 }
1437 if (!(fFlags & X86_PTE_PAE_NX) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
1438 else
1439 {
1440 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
1441 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1442 }
1443 GCPhys |= GCPtrPC & PAGE_OFFSET_MASK;
1444 /** @todo Check reserved bits and such stuff. PGM is better at doing
1445 * that, so do it when implementing the guest virtual address
1446 * TLB... */
1447
1448 /*
1449 * Read the bytes at this address.
1450 */
1451 uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
1452 if (cbToTryRead > cbLeftOnPage)
1453 cbToTryRead = cbLeftOnPage;
1454 if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
1455 cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
1456
1457 if (!pVCpu->iem.s.fBypassHandlers)
1458 {
1459 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
1460 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1461 { /* likely */ }
1462 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1463 {
1464 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1465 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1466 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1467 }
1468 else
1469 {
1470 Log((RT_SUCCESS(rcStrict)
1471 ? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1472 : "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1473 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1474 return rcStrict;
1475 }
1476 }
1477 else
1478 {
1479 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
1480 if (RT_SUCCESS(rc))
1481 { /* likely */ }
1482 else
1483 {
1484 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
1485 GCPtrPC, GCPhys, rc, cbToTryRead));
1486 return rc;
1487 }
1488 }
1489 pVCpu->iem.s.cbOpcode = cbToTryRead;
1490#endif /* !IEM_WITH_CODE_TLB */
1491 return VINF_SUCCESS;
1492}
1493
1494
1495/**
1496 * Invalidates the IEM TLBs.
1497 *
1498 * This is called internally as well as by PGM when moving GC mappings.
1499 *
1500 * @returns
1501 * @param pVCpu The cross context virtual CPU structure of the calling
1502 * thread.
1503 * @param fVmm Set when PGM calls us with a remapping.
1504 */
1505VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu, bool fVmm)
1506{
1507#ifdef IEM_WITH_CODE_TLB
1508 pVCpu->iem.s.cbInstrBufTotal = 0;
1509 pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR;
1510 if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0)
1511 { /* very likely */ }
1512 else
1513 {
1514 pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR;
1515 unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
1516 while (i-- > 0)
1517 pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0;
1518 }
1519#endif
1520
1521#ifdef IEM_WITH_DATA_TLB
1522 pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR;
1523 if (pVCpu->iem.s.DataTlb.uTlbRevision != 0)
1524 { /* very likely */ }
1525 else
1526 {
1527 pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR;
1528 unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
1529 while (i-- > 0)
1530 pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0;
1531 }
1532#endif
1533 NOREF(pVCpu); NOREF(fVmm);
1534}
1535
1536
1537/**
1538 * Invalidates a page in the TLBs.
1539 *
1540 * @param pVCpu The cross context virtual CPU structure of the calling
1541 * thread.
1542 * @param GCPtr The address of the page to invalidate
1543 */
1544VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr)
1545{
1546#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
1547 GCPtr = GCPtr >> X86_PAGE_SHIFT;
1548 AssertCompile(RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries) == 256);
1549 AssertCompile(RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries) == 256);
1550 uintptr_t idx = (uint8_t)GCPtr;
1551
1552# ifdef IEM_WITH_CODE_TLB
1553 if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.CodeTlb.uTlbRevision))
1554 {
1555 pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0;
1556 if (GCPtr == (pVCpu->iem.s.uInstrBufPc >> X86_PAGE_SHIFT))
1557 pVCpu->iem.s.cbInstrBufTotal = 0;
1558 }
1559# endif
1560
1561# ifdef IEM_WITH_DATA_TLB
1562 if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.DataTlb.uTlbRevision))
1563 pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0;
1564# endif
1565#else
1566 NOREF(pVCpu); NOREF(GCPtr);
1567#endif
1568}
1569
1570
1571/**
1572 * Invalidates the host physical aspects of the IEM TLBs.
1573 *
1574 * This is called internally as well as by PGM when moving GC mappings.
1575 *
1576 * @param pVCpu The cross context virtual CPU structure of the calling
1577 * thread.
1578 */
1579VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu)
1580{
1581#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
1582 /* Note! This probably won't end up looking exactly like this, but it give an idea... */
1583
1584# ifdef IEM_WITH_CODE_TLB
1585 pVCpu->iem.s.cbInstrBufTotal = 0;
1586# endif
1587 uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;
1588 if (uTlbPhysRev != 0)
1589 {
1590 pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;
1591 pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;
1592 }
1593 else
1594 {
1595 pVCpu->iem.s.CodeTlb.uTlbPhysRev = IEMTLB_PHYS_REV_INCR;
1596 pVCpu->iem.s.DataTlb.uTlbPhysRev = IEMTLB_PHYS_REV_INCR;
1597
1598 unsigned i;
1599# ifdef IEM_WITH_CODE_TLB
1600 i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
1601 while (i-- > 0)
1602 {
1603 pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3 = NULL;
1604 pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~(IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PHYS_REV);
1605 }
1606# endif
1607# ifdef IEM_WITH_DATA_TLB
1608 i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
1609 while (i-- > 0)
1610 {
1611 pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3 = NULL;
1612 pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~(IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PHYS_REV);
1613 }
1614# endif
1615 }
1616#else
1617 NOREF(pVCpu);
1618#endif
1619}
1620
1621
1622/**
1623 * Invalidates the host physical aspects of the IEM TLBs.
1624 *
1625 * This is called internally as well as by PGM when moving GC mappings.
1626 *
1627 * @param pVM The cross context VM structure.
1628 *
1629 * @remarks Caller holds the PGM lock.
1630 */
1631VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVM pVM)
1632{
1633 RT_NOREF_PV(pVM);
1634}
1635
1636#ifdef IEM_WITH_CODE_TLB
1637
1638/**
1639 * Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
1640 * failure and jumps.
1641 *
1642 * We end up here for a number of reasons:
1643 * - pbInstrBuf isn't yet initialized.
1644 * - Advancing beyond the buffer boundrary (e.g. cross page).
1645 * - Advancing beyond the CS segment limit.
1646 * - Fetching from non-mappable page (e.g. MMIO).
1647 *
1648 * @param pVCpu The cross context virtual CPU structure of the
1649 * calling thread.
1650 * @param pvDst Where to return the bytes.
1651 * @param cbDst Number of bytes to read.
1652 *
1653 * @todo Make cbDst = 0 a way of initializing pbInstrBuf?
1654 */
1655IEM_STATIC void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst)
1656{
1657#ifdef IN_RING3
1658 for (;;)
1659 {
1660 Assert(cbDst <= 8);
1661 uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
1662
1663 /*
1664 * We might have a partial buffer match, deal with that first to make the
1665 * rest simpler. This is the first part of the cross page/buffer case.
1666 */
1667 if (pVCpu->iem.s.pbInstrBuf != NULL)
1668 {
1669 if (offBuf < pVCpu->iem.s.cbInstrBuf)
1670 {
1671 Assert(offBuf + cbDst > pVCpu->iem.s.cbInstrBuf);
1672 uint32_t const cbCopy = pVCpu->iem.s.cbInstrBuf - pVCpu->iem.s.offInstrNextByte;
1673 memcpy(pvDst, &pVCpu->iem.s.pbInstrBuf[offBuf], cbCopy);
1674
1675 cbDst -= cbCopy;
1676 pvDst = (uint8_t *)pvDst + cbCopy;
1677 offBuf += cbCopy;
1678 pVCpu->iem.s.offInstrNextByte += offBuf;
1679 }
1680 }
1681
1682 /*
1683 * Check segment limit, figuring how much we're allowed to access at this point.
1684 *
1685 * We will fault immediately if RIP is past the segment limit / in non-canonical
1686 * territory. If we do continue, there are one or more bytes to read before we
1687 * end up in trouble and we need to do that first before faulting.
1688 */
1689 RTGCPTR GCPtrFirst;
1690 uint32_t cbMaxRead;
1691 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1692 {
1693 GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
1694 if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
1695 { /* likely */ }
1696 else
1697 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
1698 cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
1699 }
1700 else
1701 {
1702 GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
1703 Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
1704 if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
1705 { /* likely */ }
1706 else
1707 iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1708 cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
1709 if (cbMaxRead != 0)
1710 { /* likely */ }
1711 else
1712 {
1713 /* Overflowed because address is 0 and limit is max. */
1714 Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1715 cbMaxRead = X86_PAGE_SIZE;
1716 }
1717 GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
1718 uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
1719 if (cbMaxRead2 < cbMaxRead)
1720 cbMaxRead = cbMaxRead2;
1721 /** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
1722 }
1723
1724 /*
1725 * Get the TLB entry for this piece of code.
1726 */
1727 uint64_t uTag = (GCPtrFirst >> X86_PAGE_SHIFT) | pVCpu->iem.s.CodeTlb.uTlbRevision;
1728 AssertCompile(RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries) == 256);
1729 PIEMTLBENTRY pTlbe = &pVCpu->iem.s.CodeTlb.aEntries[(uint8_t)uTag];
1730 if (pTlbe->uTag == uTag)
1731 {
1732 /* likely when executing lots of code, otherwise unlikely */
1733# ifdef VBOX_WITH_STATISTICS
1734 pVCpu->iem.s.CodeTlb.cTlbHits++;
1735# endif
1736 }
1737 else
1738 {
1739 pVCpu->iem.s.CodeTlb.cTlbMisses++;
1740 RTGCPHYS GCPhys;
1741 uint64_t fFlags;
1742 int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &fFlags, &GCPhys);
1743 if (RT_FAILURE(rc))
1744 {
1745 Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
1746 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, rc);
1747 }
1748
1749 AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
1750 pTlbe->uTag = uTag;
1751 pTlbe->fFlagsAndPhysRev = (~fFlags & (X86_PTE_US | X86_PTE_RW | X86_PTE_D)) | (fFlags >> X86_PTE_PAE_BIT_NX);
1752 pTlbe->GCPhys = GCPhys;
1753 pTlbe->pbMappingR3 = NULL;
1754 }
1755
1756 /*
1757 * Check TLB page table level access flags.
1758 */
1759 if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_EXEC))
1760 {
1761 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && pVCpu->iem.s.uCpl == 3)
1762 {
1763 Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
1764 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1765 }
1766 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
1767 {
1768 Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
1769 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1770 }
1771 }
1772
1773 /*
1774 * Look up the physical page info if necessary.
1775 */
1776 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1777 { /* not necessary */ }
1778 else
1779 {
1780 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
1781 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
1782 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
1783 pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
1784 | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE);
1785 int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
1786 &pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
1787 AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
1788 }
1789
1790# if defined(IN_RING3) || (defined(IN_RING0) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE))
1791 /*
1792 * Try do a direct read using the pbMappingR3 pointer.
1793 */
1794 if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ))
1795 == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1796 {
1797 uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
1798 pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead;
1799 if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
1800 {
1801 pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(15, cbMaxRead);
1802 pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
1803 }
1804 else
1805 {
1806 uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
1807 Assert(cbInstr < cbMaxRead);
1808 pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
1809 pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
1810 }
1811 if (cbDst <= cbMaxRead)
1812 {
1813 pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
1814 pVCpu->iem.s.uInstrBufPc = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
1815 pVCpu->iem.s.pbInstrBuf = pTlbe->pbMappingR3;
1816 memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
1817 return;
1818 }
1819 pVCpu->iem.s.pbInstrBuf = NULL;
1820
1821 memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
1822 pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
1823 }
1824 else
1825# endif
1826#if 0
1827 /*
1828 * If there is no special read handling, so we can read a bit more and
1829 * put it in the prefetch buffer.
1830 */
1831 if ( cbDst < cbMaxRead
1832 && (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1833 {
1834 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
1835 &pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
1836 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1837 { /* likely */ }
1838 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1839 {
1840 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1841 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1842 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1843 AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICRC_VAL(rcStrict)));
1844 }
1845 else
1846 {
1847 Log((RT_SUCCESS(rcStrict)
1848 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1849 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1850 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1851 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1852 }
1853 }
1854 /*
1855 * Special read handling, so only read exactly what's needed.
1856 * This is a highly unlikely scenario.
1857 */
1858 else
1859#endif
1860 {
1861 pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++;
1862 uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
1863 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
1864 pvDst, cbToRead, PGMACCESSORIGIN_IEM);
1865 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1866 { /* likely */ }
1867 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1868 {
1869 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1870 GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
1871 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1872 AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict)));
1873 }
1874 else
1875 {
1876 Log((RT_SUCCESS(rcStrict)
1877 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1878 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1879 GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
1880 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1881 }
1882 pVCpu->iem.s.offInstrNextByte = offBuf + cbToRead;
1883 if (cbToRead == cbDst)
1884 return;
1885 }
1886
1887 /*
1888 * More to read, loop.
1889 */
1890 cbDst -= cbMaxRead;
1891 pvDst = (uint8_t *)pvDst + cbMaxRead;
1892 }
1893#else
1894 RT_NOREF(pvDst, cbDst);
1895 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VERR_INTERNAL_ERROR);
1896#endif
1897}
1898
1899#else
1900
1901/**
1902 * Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
1903 * exception if it fails.
1904 *
1905 * @returns Strict VBox status code.
1906 * @param pVCpu The cross context virtual CPU structure of the
1907 * calling thread.
1908 * @param cbMin The minimum number of bytes relative offOpcode
1909 * that must be read.
1910 */
1911IEM_STATIC VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin)
1912{
1913 /*
1914 * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
1915 *
1916 * First translate CS:rIP to a physical address.
1917 */
1918 uint8_t cbLeft = pVCpu->iem.s.cbOpcode - pVCpu->iem.s.offOpcode; Assert(cbLeft < cbMin);
1919 uint32_t cbToTryRead;
1920 RTGCPTR GCPtrNext;
1921 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1922 {
1923 cbToTryRead = PAGE_SIZE;
1924 GCPtrNext = pVCpu->cpum.GstCtx.rip + pVCpu->iem.s.cbOpcode;
1925 if (!IEM_IS_CANONICAL(GCPtrNext))
1926 return iemRaiseGeneralProtectionFault0(pVCpu);
1927 }
1928 else
1929 {
1930 uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
1931 Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
1932 GCPtrNext32 += pVCpu->iem.s.cbOpcode;
1933 if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
1934 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1935 cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
1936 if (!cbToTryRead) /* overflowed */
1937 {
1938 Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1939 cbToTryRead = UINT32_MAX;
1940 /** @todo check out wrapping around the code segment. */
1941 }
1942 if (cbToTryRead < cbMin - cbLeft)
1943 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1944 GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
1945 }
1946
1947 /* Only read up to the end of the page, and make sure we don't read more
1948 than the opcode buffer can hold. */
1949 uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrNext & PAGE_OFFSET_MASK);
1950 if (cbToTryRead > cbLeftOnPage)
1951 cbToTryRead = cbLeftOnPage;
1952 if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode)
1953 cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode;
1954/** @todo r=bird: Convert assertion into undefined opcode exception? */
1955 Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
1956
1957 RTGCPHYS GCPhys;
1958 uint64_t fFlags;
1959 int rc = PGMGstGetPage(pVCpu, GCPtrNext, &fFlags, &GCPhys);
1960 if (RT_FAILURE(rc))
1961 {
1962 Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
1963 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
1964 }
1965 if (!(fFlags & X86_PTE_US) && pVCpu->iem.s.uCpl == 3)
1966 {
1967 Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
1968 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1969 }
1970 if ((fFlags & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
1971 {
1972 Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
1973 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1974 }
1975 GCPhys |= GCPtrNext & PAGE_OFFSET_MASK;
1976 Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pVCpu->iem.s.cbOpcode));
1977 /** @todo Check reserved bits and such stuff. PGM is better at doing
1978 * that, so do it when implementing the guest virtual address
1979 * TLB... */
1980
1981 /*
1982 * Read the bytes at this address.
1983 *
1984 * We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
1985 * and since PATM should only patch the start of an instruction there
1986 * should be no need to check again here.
1987 */
1988 if (!pVCpu->iem.s.fBypassHandlers)
1989 {
1990 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode],
1991 cbToTryRead, PGMACCESSORIGIN_IEM);
1992 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1993 { /* likely */ }
1994 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1995 {
1996 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1997 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1998 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1999 }
2000 else
2001 {
2002 Log((RT_SUCCESS(rcStrict)
2003 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
2004 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
2005 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
2006 return rcStrict;
2007 }
2008 }
2009 else
2010 {
2011 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode], GCPhys, cbToTryRead);
2012 if (RT_SUCCESS(rc))
2013 { /* likely */ }
2014 else
2015 {
2016 Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
2017 return rc;
2018 }
2019 }
2020 pVCpu->iem.s.cbOpcode += cbToTryRead;
2021 Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
2022
2023 return VINF_SUCCESS;
2024}
2025
2026#endif /* !IEM_WITH_CODE_TLB */
2027#ifndef IEM_WITH_SETJMP
2028
2029/**
2030 * Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
2031 *
2032 * @returns Strict VBox status code.
2033 * @param pVCpu The cross context virtual CPU structure of the
2034 * calling thread.
2035 * @param pb Where to return the opcode byte.
2036 */
2037DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb)
2038{
2039 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
2040 if (rcStrict == VINF_SUCCESS)
2041 {
2042 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2043 *pb = pVCpu->iem.s.abOpcode[offOpcode];
2044 pVCpu->iem.s.offOpcode = offOpcode + 1;
2045 }
2046 else
2047 *pb = 0;
2048 return rcStrict;
2049}
2050
2051
2052/**
2053 * Fetches the next opcode byte.
2054 *
2055 * @returns Strict VBox status code.
2056 * @param pVCpu The cross context virtual CPU structure of the
2057 * calling thread.
2058 * @param pu8 Where to return the opcode byte.
2059 */
2060DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8)
2061{
2062 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2063 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2064 {
2065 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2066 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
2067 return VINF_SUCCESS;
2068 }
2069 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
2070}
2071
2072#else /* IEM_WITH_SETJMP */
2073
2074/**
2075 * Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
2076 *
2077 * @returns The opcode byte.
2078 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2079 */
2080DECL_NO_INLINE(IEM_STATIC, uint8_t) iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu)
2081{
2082# ifdef IEM_WITH_CODE_TLB
2083 uint8_t u8;
2084 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
2085 return u8;
2086# else
2087 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
2088 if (rcStrict == VINF_SUCCESS)
2089 return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
2090 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
2091# endif
2092}
2093
2094
2095/**
2096 * Fetches the next opcode byte, longjmp on error.
2097 *
2098 * @returns The opcode byte.
2099 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2100 */
2101DECLINLINE(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu)
2102{
2103# ifdef IEM_WITH_CODE_TLB
2104 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2105 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2106 if (RT_LIKELY( pbBuf != NULL
2107 && offBuf < pVCpu->iem.s.cbInstrBuf))
2108 {
2109 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
2110 return pbBuf[offBuf];
2111 }
2112# else
2113 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
2114 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2115 {
2116 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2117 return pVCpu->iem.s.abOpcode[offOpcode];
2118 }
2119# endif
2120 return iemOpcodeGetNextU8SlowJmp(pVCpu);
2121}
2122
2123#endif /* IEM_WITH_SETJMP */
2124
2125/**
2126 * Fetches the next opcode byte, returns automatically on failure.
2127 *
2128 * @param a_pu8 Where to return the opcode byte.
2129 * @remark Implicitly references pVCpu.
2130 */
2131#ifndef IEM_WITH_SETJMP
2132# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
2133 do \
2134 { \
2135 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
2136 if (rcStrict2 == VINF_SUCCESS) \
2137 { /* likely */ } \
2138 else \
2139 return rcStrict2; \
2140 } while (0)
2141#else
2142# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
2143#endif /* IEM_WITH_SETJMP */
2144
2145
2146#ifndef IEM_WITH_SETJMP
2147/**
2148 * Fetches the next signed byte from the opcode stream.
2149 *
2150 * @returns Strict VBox status code.
2151 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2152 * @param pi8 Where to return the signed byte.
2153 */
2154DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8)
2155{
2156 return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
2157}
2158#endif /* !IEM_WITH_SETJMP */
2159
2160
2161/**
2162 * Fetches the next signed byte from the opcode stream, returning automatically
2163 * on failure.
2164 *
2165 * @param a_pi8 Where to return the signed byte.
2166 * @remark Implicitly references pVCpu.
2167 */
2168#ifndef IEM_WITH_SETJMP
2169# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
2170 do \
2171 { \
2172 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
2173 if (rcStrict2 != VINF_SUCCESS) \
2174 return rcStrict2; \
2175 } while (0)
2176#else /* IEM_WITH_SETJMP */
2177# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2178
2179#endif /* IEM_WITH_SETJMP */
2180
2181#ifndef IEM_WITH_SETJMP
2182
2183/**
2184 * Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
2185 *
2186 * @returns Strict VBox status code.
2187 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2188 * @param pu16 Where to return the opcode dword.
2189 */
2190DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16)
2191{
2192 uint8_t u8;
2193 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
2194 if (rcStrict == VINF_SUCCESS)
2195 *pu16 = (int8_t)u8;
2196 return rcStrict;
2197}
2198
2199
2200/**
2201 * Fetches the next signed byte from the opcode stream, extending it to
2202 * unsigned 16-bit.
2203 *
2204 * @returns Strict VBox status code.
2205 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2206 * @param pu16 Where to return the unsigned word.
2207 */
2208DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16)
2209{
2210 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2211 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
2212 return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
2213
2214 *pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
2215 pVCpu->iem.s.offOpcode = offOpcode + 1;
2216 return VINF_SUCCESS;
2217}
2218
2219#endif /* !IEM_WITH_SETJMP */
2220
2221/**
2222 * Fetches the next signed byte from the opcode stream and sign-extending it to
2223 * a word, returning automatically on failure.
2224 *
2225 * @param a_pu16 Where to return the word.
2226 * @remark Implicitly references pVCpu.
2227 */
2228#ifndef IEM_WITH_SETJMP
2229# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
2230 do \
2231 { \
2232 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
2233 if (rcStrict2 != VINF_SUCCESS) \
2234 return rcStrict2; \
2235 } while (0)
2236#else
2237# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2238#endif
2239
2240#ifndef IEM_WITH_SETJMP
2241
2242/**
2243 * Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
2244 *
2245 * @returns Strict VBox status code.
2246 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2247 * @param pu32 Where to return the opcode dword.
2248 */
2249DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32)
2250{
2251 uint8_t u8;
2252 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
2253 if (rcStrict == VINF_SUCCESS)
2254 *pu32 = (int8_t)u8;
2255 return rcStrict;
2256}
2257
2258
2259/**
2260 * Fetches the next signed byte from the opcode stream, extending it to
2261 * unsigned 32-bit.
2262 *
2263 * @returns Strict VBox status code.
2264 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2265 * @param pu32 Where to return the unsigned dword.
2266 */
2267DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32)
2268{
2269 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2270 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
2271 return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
2272
2273 *pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
2274 pVCpu->iem.s.offOpcode = offOpcode + 1;
2275 return VINF_SUCCESS;
2276}
2277
2278#endif /* !IEM_WITH_SETJMP */
2279
2280/**
2281 * Fetches the next signed byte from the opcode stream and sign-extending it to
2282 * a word, returning automatically on failure.
2283 *
2284 * @param a_pu32 Where to return the word.
2285 * @remark Implicitly references pVCpu.
2286 */
2287#ifndef IEM_WITH_SETJMP
2288#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
2289 do \
2290 { \
2291 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
2292 if (rcStrict2 != VINF_SUCCESS) \
2293 return rcStrict2; \
2294 } while (0)
2295#else
2296# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2297#endif
2298
2299#ifndef IEM_WITH_SETJMP
2300
2301/**
2302 * Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
2303 *
2304 * @returns Strict VBox status code.
2305 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2306 * @param pu64 Where to return the opcode qword.
2307 */
2308DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
2309{
2310 uint8_t u8;
2311 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
2312 if (rcStrict == VINF_SUCCESS)
2313 *pu64 = (int8_t)u8;
2314 return rcStrict;
2315}
2316
2317
2318/**
2319 * Fetches the next signed byte from the opcode stream, extending it to
2320 * unsigned 64-bit.
2321 *
2322 * @returns Strict VBox status code.
2323 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2324 * @param pu64 Where to return the unsigned qword.
2325 */
2326DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
2327{
2328 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2329 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
2330 return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
2331
2332 *pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
2333 pVCpu->iem.s.offOpcode = offOpcode + 1;
2334 return VINF_SUCCESS;
2335}
2336
2337#endif /* !IEM_WITH_SETJMP */
2338
2339
2340/**
2341 * Fetches the next signed byte from the opcode stream and sign-extending it to
2342 * a word, returning automatically on failure.
2343 *
2344 * @param a_pu64 Where to return the word.
2345 * @remark Implicitly references pVCpu.
2346 */
2347#ifndef IEM_WITH_SETJMP
2348# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
2349 do \
2350 { \
2351 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
2352 if (rcStrict2 != VINF_SUCCESS) \
2353 return rcStrict2; \
2354 } while (0)
2355#else
2356# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2357#endif
2358
2359
2360#ifndef IEM_WITH_SETJMP
2361/**
2362 * Fetches the next opcode byte.
2363 *
2364 * @returns Strict VBox status code.
2365 * @param pVCpu The cross context virtual CPU structure of the
2366 * calling thread.
2367 * @param pu8 Where to return the opcode byte.
2368 */
2369DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8)
2370{
2371 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2372 pVCpu->iem.s.offModRm = offOpcode;
2373 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2374 {
2375 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2376 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
2377 return VINF_SUCCESS;
2378 }
2379 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
2380}
2381#else /* IEM_WITH_SETJMP */
2382/**
2383 * Fetches the next opcode byte, longjmp on error.
2384 *
2385 * @returns The opcode byte.
2386 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2387 */
2388DECLINLINE(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu)
2389{
2390# ifdef IEM_WITH_CODE_TLB
2391 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2392 pVCpu->iem.s.offModRm = offBuf;
2393 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2394 if (RT_LIKELY( pbBuf != NULL
2395 && offBuf < pVCpu->iem.s.cbInstrBuf))
2396 {
2397 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
2398 return pbBuf[offBuf];
2399 }
2400# else
2401 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
2402 pVCpu->iem.s.offModRm = offOpcode;
2403 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2404 {
2405 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2406 return pVCpu->iem.s.abOpcode[offOpcode];
2407 }
2408# endif
2409 return iemOpcodeGetNextU8SlowJmp(pVCpu);
2410}
2411#endif /* IEM_WITH_SETJMP */
2412
2413/**
2414 * Fetches the next opcode byte, which is a ModR/M byte, returns automatically
2415 * on failure.
2416 *
2417 * Will note down the position of the ModR/M byte for VT-x exits.
2418 *
2419 * @param a_pbRm Where to return the RM opcode byte.
2420 * @remark Implicitly references pVCpu.
2421 */
2422#ifndef IEM_WITH_SETJMP
2423# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
2424 do \
2425 { \
2426 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
2427 if (rcStrict2 == VINF_SUCCESS) \
2428 { /* likely */ } \
2429 else \
2430 return rcStrict2; \
2431 } while (0)
2432#else
2433# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
2434#endif /* IEM_WITH_SETJMP */
2435
2436
2437#ifndef IEM_WITH_SETJMP
2438
2439/**
2440 * Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
2441 *
2442 * @returns Strict VBox status code.
2443 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2444 * @param pu16 Where to return the opcode word.
2445 */
2446DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16)
2447{
2448 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2449 if (rcStrict == VINF_SUCCESS)
2450 {
2451 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2452# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2453 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2454# else
2455 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2456# endif
2457 pVCpu->iem.s.offOpcode = offOpcode + 2;
2458 }
2459 else
2460 *pu16 = 0;
2461 return rcStrict;
2462}
2463
2464
2465/**
2466 * Fetches the next opcode word.
2467 *
2468 * @returns Strict VBox status code.
2469 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2470 * @param pu16 Where to return the opcode word.
2471 */
2472DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16)
2473{
2474 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2475 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
2476 {
2477 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
2478# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2479 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2480# else
2481 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2482# endif
2483 return VINF_SUCCESS;
2484 }
2485 return iemOpcodeGetNextU16Slow(pVCpu, pu16);
2486}
2487
2488#else /* IEM_WITH_SETJMP */
2489
2490/**
2491 * Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
2492 *
2493 * @returns The opcode word.
2494 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2495 */
2496DECL_NO_INLINE(IEM_STATIC, uint16_t) iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu)
2497{
2498# ifdef IEM_WITH_CODE_TLB
2499 uint16_t u16;
2500 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
2501 return u16;
2502# else
2503 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2504 if (rcStrict == VINF_SUCCESS)
2505 {
2506 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2507 pVCpu->iem.s.offOpcode += 2;
2508# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2509 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2510# else
2511 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2512# endif
2513 }
2514 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
2515# endif
2516}
2517
2518
2519/**
2520 * Fetches the next opcode word, longjmp on error.
2521 *
2522 * @returns The opcode word.
2523 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2524 */
2525DECLINLINE(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu)
2526{
2527# ifdef IEM_WITH_CODE_TLB
2528 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2529 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2530 if (RT_LIKELY( pbBuf != NULL
2531 && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
2532 {
2533 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
2534# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2535 return *(uint16_t const *)&pbBuf[offBuf];
2536# else
2537 return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
2538# endif
2539 }
2540# else
2541 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2542 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
2543 {
2544 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
2545# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2546 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2547# else
2548 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2549# endif
2550 }
2551# endif
2552 return iemOpcodeGetNextU16SlowJmp(pVCpu);
2553}
2554
2555#endif /* IEM_WITH_SETJMP */
2556
2557
2558/**
2559 * Fetches the next opcode word, returns automatically on failure.
2560 *
2561 * @param a_pu16 Where to return the opcode word.
2562 * @remark Implicitly references pVCpu.
2563 */
2564#ifndef IEM_WITH_SETJMP
2565# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
2566 do \
2567 { \
2568 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
2569 if (rcStrict2 != VINF_SUCCESS) \
2570 return rcStrict2; \
2571 } while (0)
2572#else
2573# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
2574#endif
2575
2576#ifndef IEM_WITH_SETJMP
2577
2578/**
2579 * Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
2580 *
2581 * @returns Strict VBox status code.
2582 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2583 * @param pu32 Where to return the opcode double word.
2584 */
2585DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32)
2586{
2587 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2588 if (rcStrict == VINF_SUCCESS)
2589 {
2590 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2591 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2592 pVCpu->iem.s.offOpcode = offOpcode + 2;
2593 }
2594 else
2595 *pu32 = 0;
2596 return rcStrict;
2597}
2598
2599
2600/**
2601 * Fetches the next opcode word, zero extending it to a double word.
2602 *
2603 * @returns Strict VBox status code.
2604 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2605 * @param pu32 Where to return the opcode double word.
2606 */
2607DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32)
2608{
2609 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2610 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
2611 return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
2612
2613 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2614 pVCpu->iem.s.offOpcode = offOpcode + 2;
2615 return VINF_SUCCESS;
2616}
2617
2618#endif /* !IEM_WITH_SETJMP */
2619
2620
2621/**
2622 * Fetches the next opcode word and zero extends it to a double word, returns
2623 * automatically on failure.
2624 *
2625 * @param a_pu32 Where to return the opcode double word.
2626 * @remark Implicitly references pVCpu.
2627 */
2628#ifndef IEM_WITH_SETJMP
2629# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
2630 do \
2631 { \
2632 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
2633 if (rcStrict2 != VINF_SUCCESS) \
2634 return rcStrict2; \
2635 } while (0)
2636#else
2637# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
2638#endif
2639
2640#ifndef IEM_WITH_SETJMP
2641
2642/**
2643 * Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
2644 *
2645 * @returns Strict VBox status code.
2646 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2647 * @param pu64 Where to return the opcode quad word.
2648 */
2649DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
2650{
2651 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2652 if (rcStrict == VINF_SUCCESS)
2653 {
2654 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2655 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2656 pVCpu->iem.s.offOpcode = offOpcode + 2;
2657 }
2658 else
2659 *pu64 = 0;
2660 return rcStrict;
2661}
2662
2663
2664/**
2665 * Fetches the next opcode word, zero extending it to a quad word.
2666 *
2667 * @returns Strict VBox status code.
2668 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2669 * @param pu64 Where to return the opcode quad word.
2670 */
2671DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
2672{
2673 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2674 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
2675 return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
2676
2677 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2678 pVCpu->iem.s.offOpcode = offOpcode + 2;
2679 return VINF_SUCCESS;
2680}
2681
2682#endif /* !IEM_WITH_SETJMP */
2683
2684/**
2685 * Fetches the next opcode word and zero extends it to a quad word, returns
2686 * automatically on failure.
2687 *
2688 * @param a_pu64 Where to return the opcode quad word.
2689 * @remark Implicitly references pVCpu.
2690 */
2691#ifndef IEM_WITH_SETJMP
2692# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
2693 do \
2694 { \
2695 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
2696 if (rcStrict2 != VINF_SUCCESS) \
2697 return rcStrict2; \
2698 } while (0)
2699#else
2700# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
2701#endif
2702
2703
2704#ifndef IEM_WITH_SETJMP
2705/**
2706 * Fetches the next signed word from the opcode stream.
2707 *
2708 * @returns Strict VBox status code.
2709 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2710 * @param pi16 Where to return the signed word.
2711 */
2712DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16)
2713{
2714 return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
2715}
2716#endif /* !IEM_WITH_SETJMP */
2717
2718
2719/**
2720 * Fetches the next signed word from the opcode stream, returning automatically
2721 * on failure.
2722 *
2723 * @param a_pi16 Where to return the signed word.
2724 * @remark Implicitly references pVCpu.
2725 */
2726#ifndef IEM_WITH_SETJMP
2727# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
2728 do \
2729 { \
2730 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
2731 if (rcStrict2 != VINF_SUCCESS) \
2732 return rcStrict2; \
2733 } while (0)
2734#else
2735# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
2736#endif
2737
2738#ifndef IEM_WITH_SETJMP
2739
2740/**
2741 * Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
2742 *
2743 * @returns Strict VBox status code.
2744 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2745 * @param pu32 Where to return the opcode dword.
2746 */
2747DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32)
2748{
2749 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
2750 if (rcStrict == VINF_SUCCESS)
2751 {
2752 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2753# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2754 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2755# else
2756 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2757 pVCpu->iem.s.abOpcode[offOpcode + 1],
2758 pVCpu->iem.s.abOpcode[offOpcode + 2],
2759 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2760# endif
2761 pVCpu->iem.s.offOpcode = offOpcode + 4;
2762 }
2763 else
2764 *pu32 = 0;
2765 return rcStrict;
2766}
2767
2768
2769/**
2770 * Fetches the next opcode dword.
2771 *
2772 * @returns Strict VBox status code.
2773 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2774 * @param pu32 Where to return the opcode double word.
2775 */
2776DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32)
2777{
2778 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2779 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
2780 {
2781 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
2782# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2783 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2784# else
2785 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2786 pVCpu->iem.s.abOpcode[offOpcode + 1],
2787 pVCpu->iem.s.abOpcode[offOpcode + 2],
2788 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2789# endif
2790 return VINF_SUCCESS;
2791 }
2792 return iemOpcodeGetNextU32Slow(pVCpu, pu32);
2793}
2794
2795#else /* !IEM_WITH_SETJMP */
2796
2797/**
2798 * Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
2799 *
2800 * @returns The opcode dword.
2801 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2802 */
2803DECL_NO_INLINE(IEM_STATIC, uint32_t) iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu)
2804{
2805# ifdef IEM_WITH_CODE_TLB
2806 uint32_t u32;
2807 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
2808 return u32;
2809# else
2810 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
2811 if (rcStrict == VINF_SUCCESS)
2812 {
2813 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2814 pVCpu->iem.s.offOpcode = offOpcode + 4;
2815# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2816 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2817# else
2818 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2819 pVCpu->iem.s.abOpcode[offOpcode + 1],
2820 pVCpu->iem.s.abOpcode[offOpcode + 2],
2821 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2822# endif
2823 }
2824 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
2825# endif
2826}
2827
2828
2829/**
2830 * Fetches the next opcode dword, longjmp on error.
2831 *
2832 * @returns The opcode dword.
2833 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2834 */
2835DECLINLINE(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu)
2836{
2837# ifdef IEM_WITH_CODE_TLB
2838 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2839 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2840 if (RT_LIKELY( pbBuf != NULL
2841 && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
2842 {
2843 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
2844# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2845 return *(uint32_t const *)&pbBuf[offBuf];
2846# else
2847 return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
2848 pbBuf[offBuf + 1],
2849 pbBuf[offBuf + 2],
2850 pbBuf[offBuf + 3]);
2851# endif
2852 }
2853# else
2854 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2855 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
2856 {
2857 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
2858# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2859 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2860# else
2861 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2862 pVCpu->iem.s.abOpcode[offOpcode + 1],
2863 pVCpu->iem.s.abOpcode[offOpcode + 2],
2864 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2865# endif
2866 }
2867# endif
2868 return iemOpcodeGetNextU32SlowJmp(pVCpu);
2869}
2870
2871#endif /* !IEM_WITH_SETJMP */
2872
2873
2874/**
2875 * Fetches the next opcode dword, returns automatically on failure.
2876 *
2877 * @param a_pu32 Where to return the opcode dword.
2878 * @remark Implicitly references pVCpu.
2879 */
2880#ifndef IEM_WITH_SETJMP
2881# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
2882 do \
2883 { \
2884 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
2885 if (rcStrict2 != VINF_SUCCESS) \
2886 return rcStrict2; \
2887 } while (0)
2888#else
2889# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
2890#endif
2891
2892#ifndef IEM_WITH_SETJMP
2893
2894/**
2895 * Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
2896 *
2897 * @returns Strict VBox status code.
2898 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2899 * @param pu64 Where to return the opcode dword.
2900 */
2901DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
2902{
2903 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
2904 if (rcStrict == VINF_SUCCESS)
2905 {
2906 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2907 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2908 pVCpu->iem.s.abOpcode[offOpcode + 1],
2909 pVCpu->iem.s.abOpcode[offOpcode + 2],
2910 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2911 pVCpu->iem.s.offOpcode = offOpcode + 4;
2912 }
2913 else
2914 *pu64 = 0;
2915 return rcStrict;
2916}
2917
2918
2919/**
2920 * Fetches the next opcode dword, zero extending it to a quad word.
2921 *
2922 * @returns Strict VBox status code.
2923 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2924 * @param pu64 Where to return the opcode quad word.
2925 */
2926DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
2927{
2928 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2929 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
2930 return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
2931
2932 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2933 pVCpu->iem.s.abOpcode[offOpcode + 1],
2934 pVCpu->iem.s.abOpcode[offOpcode + 2],
2935 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2936 pVCpu->iem.s.offOpcode = offOpcode + 4;
2937 return VINF_SUCCESS;
2938}
2939
2940#endif /* !IEM_WITH_SETJMP */
2941
2942
2943/**
2944 * Fetches the next opcode dword and zero extends it to a quad word, returns
2945 * automatically on failure.
2946 *
2947 * @param a_pu64 Where to return the opcode quad word.
2948 * @remark Implicitly references pVCpu.
2949 */
2950#ifndef IEM_WITH_SETJMP
2951# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
2952 do \
2953 { \
2954 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
2955 if (rcStrict2 != VINF_SUCCESS) \
2956 return rcStrict2; \
2957 } while (0)
2958#else
2959# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
2960#endif
2961
2962
2963#ifndef IEM_WITH_SETJMP
2964/**
2965 * Fetches the next signed double word from the opcode stream.
2966 *
2967 * @returns Strict VBox status code.
2968 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2969 * @param pi32 Where to return the signed double word.
2970 */
2971DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32)
2972{
2973 return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
2974}
2975#endif
2976
2977/**
2978 * Fetches the next signed double word from the opcode stream, returning
2979 * automatically on failure.
2980 *
2981 * @param a_pi32 Where to return the signed double word.
2982 * @remark Implicitly references pVCpu.
2983 */
2984#ifndef IEM_WITH_SETJMP
2985# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
2986 do \
2987 { \
2988 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
2989 if (rcStrict2 != VINF_SUCCESS) \
2990 return rcStrict2; \
2991 } while (0)
2992#else
2993# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
2994#endif
2995
2996#ifndef IEM_WITH_SETJMP
2997
2998/**
2999 * Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
3000 *
3001 * @returns Strict VBox status code.
3002 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3003 * @param pu64 Where to return the opcode qword.
3004 */
3005DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
3006{
3007 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
3008 if (rcStrict == VINF_SUCCESS)
3009 {
3010 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
3011 *pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3012 pVCpu->iem.s.abOpcode[offOpcode + 1],
3013 pVCpu->iem.s.abOpcode[offOpcode + 2],
3014 pVCpu->iem.s.abOpcode[offOpcode + 3]);
3015 pVCpu->iem.s.offOpcode = offOpcode + 4;
3016 }
3017 else
3018 *pu64 = 0;
3019 return rcStrict;
3020}
3021
3022
3023/**
3024 * Fetches the next opcode dword, sign extending it into a quad word.
3025 *
3026 * @returns Strict VBox status code.
3027 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3028 * @param pu64 Where to return the opcode quad word.
3029 */
3030DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
3031{
3032 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
3033 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
3034 return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
3035
3036 int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3037 pVCpu->iem.s.abOpcode[offOpcode + 1],
3038 pVCpu->iem.s.abOpcode[offOpcode + 2],
3039 pVCpu->iem.s.abOpcode[offOpcode + 3]);
3040 *pu64 = i32;
3041 pVCpu->iem.s.offOpcode = offOpcode + 4;
3042 return VINF_SUCCESS;
3043}
3044
3045#endif /* !IEM_WITH_SETJMP */
3046
3047
3048/**
3049 * Fetches the next opcode double word and sign extends it to a quad word,
3050 * returns automatically on failure.
3051 *
3052 * @param a_pu64 Where to return the opcode quad word.
3053 * @remark Implicitly references pVCpu.
3054 */
3055#ifndef IEM_WITH_SETJMP
3056# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
3057 do \
3058 { \
3059 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
3060 if (rcStrict2 != VINF_SUCCESS) \
3061 return rcStrict2; \
3062 } while (0)
3063#else
3064# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
3065#endif
3066
3067#ifndef IEM_WITH_SETJMP
3068
3069/**
3070 * Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
3071 *
3072 * @returns Strict VBox status code.
3073 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3074 * @param pu64 Where to return the opcode qword.
3075 */
3076DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
3077{
3078 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
3079 if (rcStrict == VINF_SUCCESS)
3080 {
3081 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
3082# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3083 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3084# else
3085 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3086 pVCpu->iem.s.abOpcode[offOpcode + 1],
3087 pVCpu->iem.s.abOpcode[offOpcode + 2],
3088 pVCpu->iem.s.abOpcode[offOpcode + 3],
3089 pVCpu->iem.s.abOpcode[offOpcode + 4],
3090 pVCpu->iem.s.abOpcode[offOpcode + 5],
3091 pVCpu->iem.s.abOpcode[offOpcode + 6],
3092 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3093# endif
3094 pVCpu->iem.s.offOpcode = offOpcode + 8;
3095 }
3096 else
3097 *pu64 = 0;
3098 return rcStrict;
3099}
3100
3101
3102/**
3103 * Fetches the next opcode qword.
3104 *
3105 * @returns Strict VBox status code.
3106 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3107 * @param pu64 Where to return the opcode qword.
3108 */
3109DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64)
3110{
3111 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
3112 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
3113 {
3114# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3115 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3116# else
3117 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3118 pVCpu->iem.s.abOpcode[offOpcode + 1],
3119 pVCpu->iem.s.abOpcode[offOpcode + 2],
3120 pVCpu->iem.s.abOpcode[offOpcode + 3],
3121 pVCpu->iem.s.abOpcode[offOpcode + 4],
3122 pVCpu->iem.s.abOpcode[offOpcode + 5],
3123 pVCpu->iem.s.abOpcode[offOpcode + 6],
3124 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3125# endif
3126 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
3127 return VINF_SUCCESS;
3128 }
3129 return iemOpcodeGetNextU64Slow(pVCpu, pu64);
3130}
3131
3132#else /* IEM_WITH_SETJMP */
3133
3134/**
3135 * Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
3136 *
3137 * @returns The opcode qword.
3138 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3139 */
3140DECL_NO_INLINE(IEM_STATIC, uint64_t) iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu)
3141{
3142# ifdef IEM_WITH_CODE_TLB
3143 uint64_t u64;
3144 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
3145 return u64;
3146# else
3147 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
3148 if (rcStrict == VINF_SUCCESS)
3149 {
3150 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
3151 pVCpu->iem.s.offOpcode = offOpcode + 8;
3152# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3153 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3154# else
3155 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3156 pVCpu->iem.s.abOpcode[offOpcode + 1],
3157 pVCpu->iem.s.abOpcode[offOpcode + 2],
3158 pVCpu->iem.s.abOpcode[offOpcode + 3],
3159 pVCpu->iem.s.abOpcode[offOpcode + 4],
3160 pVCpu->iem.s.abOpcode[offOpcode + 5],
3161 pVCpu->iem.s.abOpcode[offOpcode + 6],
3162 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3163# endif
3164 }
3165 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
3166# endif
3167}
3168
3169
3170/**
3171 * Fetches the next opcode qword, longjmp on error.
3172 *
3173 * @returns The opcode qword.
3174 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3175 */
3176DECLINLINE(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu)
3177{
3178# ifdef IEM_WITH_CODE_TLB
3179 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
3180 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
3181 if (RT_LIKELY( pbBuf != NULL
3182 && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
3183 {
3184 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
3185# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3186 return *(uint64_t const *)&pbBuf[offBuf];
3187# else
3188 return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
3189 pbBuf[offBuf + 1],
3190 pbBuf[offBuf + 2],
3191 pbBuf[offBuf + 3],
3192 pbBuf[offBuf + 4],
3193 pbBuf[offBuf + 5],
3194 pbBuf[offBuf + 6],
3195 pbBuf[offBuf + 7]);
3196# endif
3197 }
3198# else
3199 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
3200 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
3201 {
3202 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
3203# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3204 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3205# else
3206 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3207 pVCpu->iem.s.abOpcode[offOpcode + 1],
3208 pVCpu->iem.s.abOpcode[offOpcode + 2],
3209 pVCpu->iem.s.abOpcode[offOpcode + 3],
3210 pVCpu->iem.s.abOpcode[offOpcode + 4],
3211 pVCpu->iem.s.abOpcode[offOpcode + 5],
3212 pVCpu->iem.s.abOpcode[offOpcode + 6],
3213 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3214# endif
3215 }
3216# endif
3217 return iemOpcodeGetNextU64SlowJmp(pVCpu);
3218}
3219
3220#endif /* IEM_WITH_SETJMP */
3221
3222/**
3223 * Fetches the next opcode quad word, returns automatically on failure.
3224 *
3225 * @param a_pu64 Where to return the opcode quad word.
3226 * @remark Implicitly references pVCpu.
3227 */
3228#ifndef IEM_WITH_SETJMP
3229# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
3230 do \
3231 { \
3232 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
3233 if (rcStrict2 != VINF_SUCCESS) \
3234 return rcStrict2; \
3235 } while (0)
3236#else
3237# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
3238#endif
3239
3240
3241/** @name Misc Worker Functions.
3242 * @{
3243 */
3244
3245/**
3246 * Gets the exception class for the specified exception vector.
3247 *
3248 * @returns The class of the specified exception.
3249 * @param uVector The exception vector.
3250 */
3251IEM_STATIC IEMXCPTCLASS iemGetXcptClass(uint8_t uVector)
3252{
3253 Assert(uVector <= X86_XCPT_LAST);
3254 switch (uVector)
3255 {
3256 case X86_XCPT_DE:
3257 case X86_XCPT_TS:
3258 case X86_XCPT_NP:
3259 case X86_XCPT_SS:
3260 case X86_XCPT_GP:
3261 case X86_XCPT_SX: /* AMD only */
3262 return IEMXCPTCLASS_CONTRIBUTORY;
3263
3264 case X86_XCPT_PF:
3265 case X86_XCPT_VE: /* Intel only */
3266 return IEMXCPTCLASS_PAGE_FAULT;
3267
3268 case X86_XCPT_DF:
3269 return IEMXCPTCLASS_DOUBLE_FAULT;
3270 }
3271 return IEMXCPTCLASS_BENIGN;
3272}
3273
3274
3275/**
3276 * Evaluates how to handle an exception caused during delivery of another event
3277 * (exception / interrupt).
3278 *
3279 * @returns How to handle the recursive exception.
3280 * @param pVCpu The cross context virtual CPU structure of the
3281 * calling thread.
3282 * @param fPrevFlags The flags of the previous event.
3283 * @param uPrevVector The vector of the previous event.
3284 * @param fCurFlags The flags of the current exception.
3285 * @param uCurVector The vector of the current exception.
3286 * @param pfXcptRaiseInfo Where to store additional information about the
3287 * exception condition. Optional.
3288 */
3289VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPUCC pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags,
3290 uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo)
3291{
3292 /*
3293 * Only CPU exceptions can be raised while delivering other events, software interrupt
3294 * (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception.
3295 */
3296 AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID);
3297 Assert(pVCpu); RT_NOREF(pVCpu);
3298 Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector));
3299
3300 IEMXCPTRAISE enmRaise = IEMXCPTRAISE_CURRENT_XCPT;
3301 IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE;
3302 if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3303 {
3304 IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector);
3305 if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN)
3306 {
3307 IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector);
3308 if ( enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT
3309 && ( enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT
3310 || enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY))
3311 {
3312 enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
3313 fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF
3314 : IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT;
3315 Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector,
3316 uCurVector, pVCpu->cpum.GstCtx.cr2));
3317 }
3318 else if ( enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY
3319 && enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY)
3320 {
3321 enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
3322 Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector));
3323 }
3324 else if ( enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT
3325 && ( enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY
3326 || enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT))
3327 {
3328 enmRaise = IEMXCPTRAISE_TRIPLE_FAULT;
3329 Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector));
3330 }
3331 }
3332 else
3333 {
3334 if (uPrevVector == X86_XCPT_NMI)
3335 {
3336 fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT;
3337 if (uCurVector == X86_XCPT_PF)
3338 {
3339 fRaiseInfo |= IEMXCPTRAISEINFO_NMI_PF;
3340 Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n"));
3341 }
3342 }
3343 else if ( uPrevVector == X86_XCPT_AC
3344 && uCurVector == X86_XCPT_AC)
3345 {
3346 enmRaise = IEMXCPTRAISE_CPU_HANG;
3347 fRaiseInfo = IEMXCPTRAISEINFO_AC_AC;
3348 Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n"));
3349 }
3350 }
3351 }
3352 else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT)
3353 {
3354 fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT;
3355 if (uCurVector == X86_XCPT_PF)
3356 fRaiseInfo |= IEMXCPTRAISEINFO_EXT_INT_PF;
3357 }
3358 else
3359 {
3360 Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT);
3361 fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT;
3362 }
3363
3364 if (pfXcptRaiseInfo)
3365 *pfXcptRaiseInfo = fRaiseInfo;
3366 return enmRaise;
3367}
3368
3369
3370/**
3371 * Enters the CPU shutdown state initiated by a triple fault or other
3372 * unrecoverable conditions.
3373 *
3374 * @returns Strict VBox status code.
3375 * @param pVCpu The cross context virtual CPU structure of the
3376 * calling thread.
3377 */
3378IEM_STATIC VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPUCC pVCpu)
3379{
3380 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3381 IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
3382
3383 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
3384 {
3385 Log2(("shutdown: Guest intercept -> #VMEXIT\n"));
3386 IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
3387 }
3388
3389 RT_NOREF(pVCpu);
3390 return VINF_EM_TRIPLE_FAULT;
3391}
3392
3393
3394/**
3395 * Validates a new SS segment.
3396 *
3397 * @returns VBox strict status code.
3398 * @param pVCpu The cross context virtual CPU structure of the
3399 * calling thread.
3400 * @param NewSS The new SS selctor.
3401 * @param uCpl The CPL to load the stack for.
3402 * @param pDesc Where to return the descriptor.
3403 */
3404IEM_STATIC VBOXSTRICTRC iemMiscValidateNewSS(PVMCPUCC pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc)
3405{
3406 /* Null selectors are not allowed (we're not called for dispatching
3407 interrupts with SS=0 in long mode). */
3408 if (!(NewSS & X86_SEL_MASK_OFF_RPL))
3409 {
3410 Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS));
3411 return iemRaiseTaskSwitchFault0(pVCpu);
3412 }
3413
3414 /** @todo testcase: check that the TSS.ssX RPL is checked. Also check when. */
3415 if ((NewSS & X86_SEL_RPL) != uCpl)
3416 {
3417 Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
3418 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3419 }
3420
3421 /*
3422 * Read the descriptor.
3423 */
3424 VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS);
3425 if (rcStrict != VINF_SUCCESS)
3426 return rcStrict;
3427
3428 /*
3429 * Perform the descriptor validation documented for LSS, POP SS and MOV SS.
3430 */
3431 if (!pDesc->Legacy.Gen.u1DescType)
3432 {
3433 Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
3434 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3435 }
3436
3437 if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
3438 || !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
3439 {
3440 Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
3441 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3442 }
3443 if (pDesc->Legacy.Gen.u2Dpl != uCpl)
3444 {
3445 Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
3446 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3447 }
3448
3449 /* Is it there? */
3450 /** @todo testcase: Is this checked before the canonical / limit check below? */
3451 if (!pDesc->Legacy.Gen.u1Present)
3452 {
3453 Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
3454 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS);
3455 }
3456
3457 return VINF_SUCCESS;
3458}
3459
3460
3461/**
3462 * Gets the correct EFLAGS regardless of whether PATM stores parts of them or
3463 * not (kind of obsolete now).
3464 *
3465 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
3466 */
3467#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
3468
3469/**
3470 * Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
3471 *
3472 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
3473 * @param a_fEfl The new EFLAGS.
3474 */
3475#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
3476
3477/** @} */
3478
3479
3480/** @name Raising Exceptions.
3481 *
3482 * @{
3483 */
3484
3485
3486/**
3487 * Loads the specified stack far pointer from the TSS.
3488 *
3489 * @returns VBox strict status code.
3490 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3491 * @param uCpl The CPL to load the stack for.
3492 * @param pSelSS Where to return the new stack segment.
3493 * @param puEsp Where to return the new stack pointer.
3494 */
3495IEM_STATIC VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPUCC pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp)
3496{
3497 VBOXSTRICTRC rcStrict;
3498 Assert(uCpl < 4);
3499
3500 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
3501 switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
3502 {
3503 /*
3504 * 16-bit TSS (X86TSS16).
3505 */
3506 case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
3507 case X86_SEL_TYPE_SYS_286_TSS_BUSY:
3508 {
3509 uint32_t off = uCpl * 4 + 2;
3510 if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit)
3511 {
3512 /** @todo check actual access pattern here. */
3513 uint32_t u32Tmp = 0; /* gcc maybe... */
3514 rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
3515 if (rcStrict == VINF_SUCCESS)
3516 {
3517 *puEsp = RT_LOWORD(u32Tmp);
3518 *pSelSS = RT_HIWORD(u32Tmp);
3519 return VINF_SUCCESS;
3520 }
3521 }
3522 else
3523 {
3524 Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
3525 rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
3526 }
3527 break;
3528 }
3529
3530 /*
3531 * 32-bit TSS (X86TSS32).
3532 */
3533 case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
3534 case X86_SEL_TYPE_SYS_386_TSS_BUSY:
3535 {
3536 uint32_t off = uCpl * 8 + 4;
3537 if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit)
3538 {
3539/** @todo check actual access pattern here. */
3540 uint64_t u64Tmp;
3541 rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
3542 if (rcStrict == VINF_SUCCESS)
3543 {
3544 *puEsp = u64Tmp & UINT32_MAX;
3545 *pSelSS = (RTSEL)(u64Tmp >> 32);
3546 return VINF_SUCCESS;
3547 }
3548 }
3549 else
3550 {
3551 Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
3552 rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
3553 }
3554 break;
3555 }
3556
3557 default:
3558 AssertFailed();
3559 rcStrict = VERR_IEM_IPE_4;
3560 break;
3561 }
3562
3563 *puEsp = 0; /* make gcc happy */
3564 *pSelSS = 0; /* make gcc happy */
3565 return rcStrict;
3566}
3567
3568
3569/**
3570 * Loads the specified stack pointer from the 64-bit TSS.
3571 *
3572 * @returns VBox strict status code.
3573 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3574 * @param uCpl The CPL to load the stack for.
3575 * @param uIst The interrupt stack table index, 0 if to use uCpl.
3576 * @param puRsp Where to return the new stack pointer.
3577 */
3578IEM_STATIC VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPUCC pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp)
3579{
3580 Assert(uCpl < 4);
3581 Assert(uIst < 8);
3582 *puRsp = 0; /* make gcc happy */
3583
3584 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
3585 AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5);
3586
3587 uint32_t off;
3588 if (uIst)
3589 off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1);
3590 else
3591 off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0);
3592 if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit)
3593 {
3594 Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit));
3595 return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
3596 }
3597
3598 return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
3599}
3600
3601
3602/**
3603 * Adjust the CPU state according to the exception being raised.
3604 *
3605 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3606 * @param u8Vector The exception that has been raised.
3607 */
3608DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPUCC pVCpu, uint8_t u8Vector)
3609{
3610 switch (u8Vector)
3611 {
3612 case X86_XCPT_DB:
3613 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
3614 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
3615 break;
3616 /** @todo Read the AMD and Intel exception reference... */
3617 }
3618}
3619
3620
3621/**
3622 * Implements exceptions and interrupts for real mode.
3623 *
3624 * @returns VBox strict status code.
3625 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3626 * @param cbInstr The number of bytes to offset rIP by in the return
3627 * address.
3628 * @param u8Vector The interrupt / exception vector number.
3629 * @param fFlags The flags.
3630 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3631 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3632 */
3633IEM_STATIC VBOXSTRICTRC
3634iemRaiseXcptOrIntInRealMode(PVMCPUCC pVCpu,
3635 uint8_t cbInstr,
3636 uint8_t u8Vector,
3637 uint32_t fFlags,
3638 uint16_t uErr,
3639 uint64_t uCr2)
3640{
3641 NOREF(uErr); NOREF(uCr2);
3642 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3643
3644 /*
3645 * Read the IDT entry.
3646 */
3647 if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
3648 {
3649 Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
3650 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3651 }
3652 RTFAR16 Idte;
3653 VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t *)&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) * u8Vector);
3654 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3655 {
3656 Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
3657 return rcStrict;
3658 }
3659
3660 /*
3661 * Push the stack frame.
3662 */
3663 uint16_t *pu16Frame;
3664 uint64_t uNewRsp;
3665 rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, (void **)&pu16Frame, &uNewRsp);
3666 if (rcStrict != VINF_SUCCESS)
3667 return rcStrict;
3668
3669 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3670#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
3671 AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
3672 if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186)
3673 fEfl |= UINT16_C(0xf000);
3674#endif
3675 pu16Frame[2] = (uint16_t)fEfl;
3676 pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel;
3677 pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
3678 rcStrict = iemMemStackPushCommitSpecial(pVCpu, pu16Frame, uNewRsp);
3679 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3680 return rcStrict;
3681
3682 /*
3683 * Load the vector address into cs:ip and make exception specific state
3684 * adjustments.
3685 */
3686 pVCpu->cpum.GstCtx.cs.Sel = Idte.sel;
3687 pVCpu->cpum.GstCtx.cs.ValidSel = Idte.sel;
3688 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3689 pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)Idte.sel << 4;
3690 /** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
3691 pVCpu->cpum.GstCtx.rip = Idte.off;
3692 fEfl &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_AC);
3693 IEMMISC_SET_EFL(pVCpu, fEfl);
3694
3695 /** @todo do we actually do this in real mode? */
3696 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3697 iemRaiseXcptAdjustState(pVCpu, u8Vector);
3698
3699 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3700}
3701
3702
3703/**
3704 * Loads a NULL data selector into when coming from V8086 mode.
3705 *
3706 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3707 * @param pSReg Pointer to the segment register.
3708 */
3709IEM_STATIC void iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
3710{
3711 pSReg->Sel = 0;
3712 pSReg->ValidSel = 0;
3713 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
3714 {
3715 /* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
3716 pSReg->Attr.u &= X86DESCATTR_DT | X86DESCATTR_TYPE | X86DESCATTR_DPL | X86DESCATTR_G | X86DESCATTR_D;
3717 pSReg->Attr.u |= X86DESCATTR_UNUSABLE;
3718 }
3719 else
3720 {
3721 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3722 /** @todo check this on AMD-V */
3723 pSReg->u64Base = 0;
3724 pSReg->u32Limit = 0;
3725 }
3726}
3727
3728
3729/**
3730 * Loads a segment selector during a task switch in V8086 mode.
3731 *
3732 * @param pSReg Pointer to the segment register.
3733 * @param uSel The selector value to load.
3734 */
3735IEM_STATIC void iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel)
3736{
3737 /* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3738 pSReg->Sel = uSel;
3739 pSReg->ValidSel = uSel;
3740 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3741 pSReg->u64Base = uSel << 4;
3742 pSReg->u32Limit = 0xffff;
3743 pSReg->Attr.u = 0xf3;
3744}
3745
3746
3747/**
3748 * Loads a NULL data selector into a selector register, both the hidden and
3749 * visible parts, in protected mode.
3750 *
3751 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3752 * @param pSReg Pointer to the segment register.
3753 * @param uRpl The RPL.
3754 */
3755IEM_STATIC void iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl)
3756{
3757 /** @todo Testcase: write a testcase checking what happends when loading a NULL
3758 * data selector in protected mode. */
3759 pSReg->Sel = uRpl;
3760 pSReg->ValidSel = uRpl;
3761 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3762 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
3763 {
3764 /* VT-x (Intel 3960x) observed doing something like this. */
3765 pSReg->Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
3766 pSReg->u32Limit = UINT32_MAX;
3767 pSReg->u64Base = 0;
3768 }
3769 else
3770 {
3771 pSReg->Attr.u = X86DESCATTR_UNUSABLE;
3772 pSReg->u32Limit = 0;
3773 pSReg->u64Base = 0;
3774 }
3775}
3776
3777
3778/**
3779 * Loads a segment selector during a task switch in protected mode.
3780 *
3781 * In this task switch scenario, we would throw \#TS exceptions rather than
3782 * \#GPs.
3783 *
3784 * @returns VBox strict status code.
3785 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3786 * @param pSReg Pointer to the segment register.
3787 * @param uSel The new selector value.
3788 *
3789 * @remarks This does _not_ handle CS or SS.
3790 * @remarks This expects pVCpu->iem.s.uCpl to be up to date.
3791 */
3792IEM_STATIC VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPUCC pVCpu, PCPUMSELREG pSReg, uint16_t uSel)
3793{
3794 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
3795
3796 /* Null data selector. */
3797 if (!(uSel & X86_SEL_MASK_OFF_RPL))
3798 {
3799 iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel);
3800 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
3801 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
3802 return VINF_SUCCESS;
3803 }
3804
3805 /* Fetch the descriptor. */
3806 IEMSELDESC Desc;
3807 VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS);
3808 if (rcStrict != VINF_SUCCESS)
3809 {
3810 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
3811 VBOXSTRICTRC_VAL(rcStrict)));
3812 return rcStrict;
3813 }
3814
3815 /* Must be a data segment or readable code segment. */
3816 if ( !Desc.Legacy.Gen.u1DescType
3817 || (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3818 {
3819 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
3820 Desc.Legacy.Gen.u4Type));
3821 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
3822 }
3823
3824 /* Check privileges for data segments and non-conforming code segments. */
3825 if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
3826 != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
3827 {
3828 /* The RPL and the new CPL must be less than or equal to the DPL. */
3829 if ( (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
3830 || (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl))
3831 {
3832 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
3833 uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3834 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
3835 }
3836 }
3837
3838 /* Is it there? */
3839 if (!Desc.Legacy.Gen.u1Present)
3840 {
3841 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
3842 return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
3843 }
3844
3845 /* The base and limit. */
3846 uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
3847 uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
3848
3849 /*
3850 * Ok, everything checked out fine. Now set the accessed bit before
3851 * committing the result into the registers.
3852 */
3853 if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3854 {
3855 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
3856 if (rcStrict != VINF_SUCCESS)
3857 return rcStrict;
3858 Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
3859 }
3860
3861 /* Commit */
3862 pSReg->Sel = uSel;
3863 pSReg->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
3864 pSReg->u32Limit = cbLimit;
3865 pSReg->u64Base = u64Base; /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */
3866 pSReg->ValidSel = uSel;
3867 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3868 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
3869 pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
3870
3871 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
3872 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
3873 return VINF_SUCCESS;
3874}
3875
3876
3877/**
3878 * Performs a task switch.
3879 *
3880 * If the task switch is the result of a JMP, CALL or IRET instruction, the
3881 * caller is responsible for performing the necessary checks (like DPL, TSS
3882 * present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
3883 * reference for JMP, CALL, IRET.
3884 *
3885 * If the task switch is the due to a software interrupt or hardware exception,
3886 * the caller is responsible for validating the TSS selector and descriptor. See
3887 * Intel Instruction reference for INT n.
3888 *
3889 * @returns VBox strict status code.
3890 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3891 * @param enmTaskSwitch The cause of the task switch.
3892 * @param uNextEip The EIP effective after the task switch.
3893 * @param fFlags The flags, see IEM_XCPT_FLAGS_XXX.
3894 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3895 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3896 * @param SelTSS The TSS selector of the new task.
3897 * @param pNewDescTSS Pointer to the new TSS descriptor.
3898 */
3899IEM_STATIC VBOXSTRICTRC
3900iemTaskSwitch(PVMCPUCC pVCpu,
3901 IEMTASKSWITCH enmTaskSwitch,
3902 uint32_t uNextEip,
3903 uint32_t fFlags,
3904 uint16_t uErr,
3905 uint64_t uCr2,
3906 RTSEL SelTSS,
3907 PIEMSELDESC pNewDescTSS)
3908{
3909 Assert(!IEM_IS_REAL_MODE(pVCpu));
3910 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
3911 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3912
3913 uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type;
3914 Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
3915 || uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
3916 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
3917 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
3918
3919 bool const fIsNewTSS386 = ( uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
3920 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
3921
3922 Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTSS,
3923 fIsNewTSS386, pVCpu->cpum.GstCtx.eip, uNextEip));
3924
3925 /* Update CR2 in case it's a page-fault. */
3926 /** @todo This should probably be done much earlier in IEM/PGM. See
3927 * @bugref{5653#c49}. */
3928 if (fFlags & IEM_XCPT_FLAGS_CR2)
3929 pVCpu->cpum.GstCtx.cr2 = uCr2;
3930
3931 /*
3932 * Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
3933 * subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
3934 */
3935 uint32_t const uNewTSSLimit = pNewDescTSS->Legacy.Gen.u16LimitLow | (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16);
3936 uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
3937 if (uNewTSSLimit < uNewTSSLimitMin)
3938 {
3939 Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n",
3940 enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin));
3941 return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
3942 }
3943
3944 /*
3945 * Task switches in VMX non-root mode always cause task switches.
3946 * The new TSS must have been read and validated (DPL, limits etc.) before a
3947 * task-switch VM-exit commences.
3948 *
3949 * See Intel spec. 25.4.2 "Treatment of Task Switches".
3950 */
3951 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3952 {
3953 Log(("iemTaskSwitch: Guest intercept (source=%u, sel=%#x) -> VM-exit.\n", enmTaskSwitch, SelTSS));
3954 IEM_VMX_VMEXIT_TASK_SWITCH_RET(pVCpu, enmTaskSwitch, SelTSS, uNextEip - pVCpu->cpum.GstCtx.eip);
3955 }
3956
3957 /*
3958 * The SVM nested-guest intercept for task-switch takes priority over all exceptions
3959 * after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept".
3960 */
3961 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
3962 {
3963 uint32_t const uExitInfo1 = SelTSS;
3964 uint32_t uExitInfo2 = uErr;
3965 switch (enmTaskSwitch)
3966 {
3967 case IEMTASKSWITCH_JUMP: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_JUMP; break;
3968 case IEMTASKSWITCH_IRET: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_IRET; break;
3969 default: break;
3970 }
3971 if (fFlags & IEM_XCPT_FLAGS_ERR)
3972 uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE;
3973 if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
3974 uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF;
3975
3976 Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2));
3977 IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2);
3978 RT_NOREF2(uExitInfo1, uExitInfo2);
3979 }
3980
3981 /*
3982 * Check the current TSS limit. The last written byte to the current TSS during the
3983 * task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
3984 * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
3985 *
3986 * The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
3987 * end up with smaller than "legal" TSS limits.
3988 */
3989 uint32_t const uCurTSSLimit = pVCpu->cpum.GstCtx.tr.u32Limit;
3990 uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29;
3991 if (uCurTSSLimit < uCurTSSLimitMin)
3992 {
3993 Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n",
3994 enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin));
3995 return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
3996 }
3997
3998 /*
3999 * Verify that the new TSS can be accessed and map it. Map only the required contents
4000 * and not the entire TSS.
4001 */
4002 void *pvNewTSS;
4003 uint32_t const cbNewTSS = uNewTSSLimitMin + 1;
4004 RTGCPTR const GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy);
4005 AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
4006 /** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
4007 * not perform correct translation if this happens. See Intel spec. 7.2.1
4008 * "Task-State Segment". */
4009 VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW);
4010 if (rcStrict != VINF_SUCCESS)
4011 {
4012 Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch,
4013 cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict)));
4014 return rcStrict;
4015 }
4016
4017 /*
4018 * Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
4019 */
4020 uint32_t u32EFlags = pVCpu->cpum.GstCtx.eflags.u32;
4021 if ( enmTaskSwitch == IEMTASKSWITCH_JUMP
4022 || enmTaskSwitch == IEMTASKSWITCH_IRET)
4023 {
4024 PX86DESC pDescCurTSS;
4025 rcStrict = iemMemMap(pVCpu, (void **)&pDescCurTSS, sizeof(*pDescCurTSS), UINT8_MAX,
4026 pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW);
4027 if (rcStrict != VINF_SUCCESS)
4028 {
4029 Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4030 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4031 return rcStrict;
4032 }
4033
4034 pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
4035 rcStrict = iemMemCommitAndUnmap(pVCpu, pDescCurTSS, IEM_ACCESS_SYS_RW);
4036 if (rcStrict != VINF_SUCCESS)
4037 {
4038 Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4039 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4040 return rcStrict;
4041 }
4042
4043 /* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
4044 if (enmTaskSwitch == IEMTASKSWITCH_IRET)
4045 {
4046 Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
4047 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
4048 u32EFlags &= ~X86_EFL_NT;
4049 }
4050 }
4051
4052 /*
4053 * Save the CPU state into the current TSS.
4054 */
4055 RTGCPTR const GCPtrCurTSS = pVCpu->cpum.GstCtx.tr.u64Base;
4056 if (GCPtrNewTSS == GCPtrCurTSS)
4057 {
4058 Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS));
4059 Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
4060 pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.eax,
4061 pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel,
4062 pVCpu->cpum.GstCtx.ldtr.Sel));
4063 }
4064 if (fIsNewTSS386)
4065 {
4066 /*
4067 * Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
4068 * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
4069 */
4070 void *pvCurTSS32;
4071 uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS32, eip);
4072 uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip);
4073 AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
4074 rcStrict = iemMemMap(pVCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW);
4075 if (rcStrict != VINF_SUCCESS)
4076 {
4077 Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
4078 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
4079 return rcStrict;
4080 }
4081
4082 /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
4083 PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS);
4084 pCurTSS32->eip = uNextEip;
4085 pCurTSS32->eflags = u32EFlags;
4086 pCurTSS32->eax = pVCpu->cpum.GstCtx.eax;
4087 pCurTSS32->ecx = pVCpu->cpum.GstCtx.ecx;
4088 pCurTSS32->edx = pVCpu->cpum.GstCtx.edx;
4089 pCurTSS32->ebx = pVCpu->cpum.GstCtx.ebx;
4090 pCurTSS32->esp = pVCpu->cpum.GstCtx.esp;
4091 pCurTSS32->ebp = pVCpu->cpum.GstCtx.ebp;
4092 pCurTSS32->esi = pVCpu->cpum.GstCtx.esi;
4093 pCurTSS32->edi = pVCpu->cpum.GstCtx.edi;
4094 pCurTSS32->es = pVCpu->cpum.GstCtx.es.Sel;
4095 pCurTSS32->cs = pVCpu->cpum.GstCtx.cs.Sel;
4096 pCurTSS32->ss = pVCpu->cpum.GstCtx.ss.Sel;
4097 pCurTSS32->ds = pVCpu->cpum.GstCtx.ds.Sel;
4098 pCurTSS32->fs = pVCpu->cpum.GstCtx.fs.Sel;
4099 pCurTSS32->gs = pVCpu->cpum.GstCtx.gs.Sel;
4100
4101 rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS32, IEM_ACCESS_SYS_RW);
4102 if (rcStrict != VINF_SUCCESS)
4103 {
4104 Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
4105 VBOXSTRICTRC_VAL(rcStrict)));
4106 return rcStrict;
4107 }
4108 }
4109 else
4110 {
4111 /*
4112 * Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
4113 */
4114 void *pvCurTSS16;
4115 uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS16, ip);
4116 uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip);
4117 AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
4118 rcStrict = iemMemMap(pVCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW);
4119 if (rcStrict != VINF_SUCCESS)
4120 {
4121 Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
4122 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
4123 return rcStrict;
4124 }
4125
4126 /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
4127 PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS);
4128 pCurTSS16->ip = uNextEip;
4129 pCurTSS16->flags = u32EFlags;
4130 pCurTSS16->ax = pVCpu->cpum.GstCtx.ax;
4131 pCurTSS16->cx = pVCpu->cpum.GstCtx.cx;
4132 pCurTSS16->dx = pVCpu->cpum.GstCtx.dx;
4133 pCurTSS16->bx = pVCpu->cpum.GstCtx.bx;
4134 pCurTSS16->sp = pVCpu->cpum.GstCtx.sp;
4135 pCurTSS16->bp = pVCpu->cpum.GstCtx.bp;
4136 pCurTSS16->si = pVCpu->cpum.GstCtx.si;
4137 pCurTSS16->di = pVCpu->cpum.GstCtx.di;
4138 pCurTSS16->es = pVCpu->cpum.GstCtx.es.Sel;
4139 pCurTSS16->cs = pVCpu->cpum.GstCtx.cs.Sel;
4140 pCurTSS16->ss = pVCpu->cpum.GstCtx.ss.Sel;
4141 pCurTSS16->ds = pVCpu->cpum.GstCtx.ds.Sel;
4142
4143 rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS16, IEM_ACCESS_SYS_RW);
4144 if (rcStrict != VINF_SUCCESS)
4145 {
4146 Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
4147 VBOXSTRICTRC_VAL(rcStrict)));
4148 return rcStrict;
4149 }
4150 }
4151
4152 /*
4153 * Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
4154 */
4155 if ( enmTaskSwitch == IEMTASKSWITCH_CALL
4156 || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
4157 {
4158 /* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
4159 PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS;
4160 pNewTSS->selPrev = pVCpu->cpum.GstCtx.tr.Sel;
4161 }
4162
4163 /*
4164 * Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
4165 * it's done further below with error handling (e.g. CR3 changes will go through PGM).
4166 */
4167 uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
4168 uint16_t uNewES, uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
4169 bool fNewDebugTrap;
4170 if (fIsNewTSS386)
4171 {
4172 PCX86TSS32 pNewTSS32 = (PCX86TSS32)pvNewTSS;
4173 uNewCr3 = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0;
4174 uNewEip = pNewTSS32->eip;
4175 uNewEflags = pNewTSS32->eflags;
4176 uNewEax = pNewTSS32->eax;
4177 uNewEcx = pNewTSS32->ecx;
4178 uNewEdx = pNewTSS32->edx;
4179 uNewEbx = pNewTSS32->ebx;
4180 uNewEsp = pNewTSS32->esp;
4181 uNewEbp = pNewTSS32->ebp;
4182 uNewEsi = pNewTSS32->esi;
4183 uNewEdi = pNewTSS32->edi;
4184 uNewES = pNewTSS32->es;
4185 uNewCS = pNewTSS32->cs;
4186 uNewSS = pNewTSS32->ss;
4187 uNewDS = pNewTSS32->ds;
4188 uNewFS = pNewTSS32->fs;
4189 uNewGS = pNewTSS32->gs;
4190 uNewLdt = pNewTSS32->selLdt;
4191 fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap);
4192 }
4193 else
4194 {
4195 PCX86TSS16 pNewTSS16 = (PCX86TSS16)pvNewTSS;
4196 uNewCr3 = 0;
4197 uNewEip = pNewTSS16->ip;
4198 uNewEflags = pNewTSS16->flags;
4199 uNewEax = UINT32_C(0xffff0000) | pNewTSS16->ax;
4200 uNewEcx = UINT32_C(0xffff0000) | pNewTSS16->cx;
4201 uNewEdx = UINT32_C(0xffff0000) | pNewTSS16->dx;
4202 uNewEbx = UINT32_C(0xffff0000) | pNewTSS16->bx;
4203 uNewEsp = UINT32_C(0xffff0000) | pNewTSS16->sp;
4204 uNewEbp = UINT32_C(0xffff0000) | pNewTSS16->bp;
4205 uNewEsi = UINT32_C(0xffff0000) | pNewTSS16->si;
4206 uNewEdi = UINT32_C(0xffff0000) | pNewTSS16->di;
4207 uNewES = pNewTSS16->es;
4208 uNewCS = pNewTSS16->cs;
4209 uNewSS = pNewTSS16->ss;
4210 uNewDS = pNewTSS16->ds;
4211 uNewFS = 0;
4212 uNewGS = 0;
4213 uNewLdt = pNewTSS16->selLdt;
4214 fNewDebugTrap = false;
4215 }
4216
4217 if (GCPtrNewTSS == GCPtrCurTSS)
4218 Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
4219 uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
4220
4221 /*
4222 * We're done accessing the new TSS.
4223 */
4224 rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewTSS, IEM_ACCESS_SYS_RW);
4225 if (rcStrict != VINF_SUCCESS)
4226 {
4227 Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
4228 return rcStrict;
4229 }
4230
4231 /*
4232 * Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
4233 */
4234 if (enmTaskSwitch != IEMTASKSWITCH_IRET)
4235 {
4236 rcStrict = iemMemMap(pVCpu, (void **)&pNewDescTSS, sizeof(*pNewDescTSS), UINT8_MAX,
4237 pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW);
4238 if (rcStrict != VINF_SUCCESS)
4239 {
4240 Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4241 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4242 return rcStrict;
4243 }
4244
4245 /* Check that the descriptor indicates the new TSS is available (not busy). */
4246 AssertMsg( pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
4247 || pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
4248 ("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type));
4249
4250 pNewDescTSS->Legacy.Gate.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
4251 rcStrict = iemMemCommitAndUnmap(pVCpu, pNewDescTSS, IEM_ACCESS_SYS_RW);
4252 if (rcStrict != VINF_SUCCESS)
4253 {
4254 Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4255 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4256 return rcStrict;
4257 }
4258 }
4259
4260 /*
4261 * From this point on, we're technically in the new task. We will defer exceptions
4262 * until the completion of the task switch but before executing any instructions in the new task.
4263 */
4264 pVCpu->cpum.GstCtx.tr.Sel = SelTSS;
4265 pVCpu->cpum.GstCtx.tr.ValidSel = SelTSS;
4266 pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
4267 pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy);
4268 pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy);
4269 pVCpu->cpum.GstCtx.tr.u64Base = X86DESC_BASE(&pNewDescTSS->Legacy);
4270 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR);
4271
4272 /* Set the busy bit in TR. */
4273 pVCpu->cpum.GstCtx.tr.Attr.n.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
4274
4275 /* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
4276 if ( enmTaskSwitch == IEMTASKSWITCH_CALL
4277 || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
4278 {
4279 uNewEflags |= X86_EFL_NT;
4280 }
4281
4282 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL; /** @todo Should we clear DR7.LE bit too? */
4283 pVCpu->cpum.GstCtx.cr0 |= X86_CR0_TS;
4284 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0);
4285
4286 pVCpu->cpum.GstCtx.eip = uNewEip;
4287 pVCpu->cpum.GstCtx.eax = uNewEax;
4288 pVCpu->cpum.GstCtx.ecx = uNewEcx;
4289 pVCpu->cpum.GstCtx.edx = uNewEdx;
4290 pVCpu->cpum.GstCtx.ebx = uNewEbx;
4291 pVCpu->cpum.GstCtx.esp = uNewEsp;
4292 pVCpu->cpum.GstCtx.ebp = uNewEbp;
4293 pVCpu->cpum.GstCtx.esi = uNewEsi;
4294 pVCpu->cpum.GstCtx.edi = uNewEdi;
4295
4296 uNewEflags &= X86_EFL_LIVE_MASK;
4297 uNewEflags |= X86_EFL_RA1_MASK;
4298 IEMMISC_SET_EFL(pVCpu, uNewEflags);
4299
4300 /*
4301 * Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
4302 * will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
4303 * due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
4304 */
4305 pVCpu->cpum.GstCtx.es.Sel = uNewES;
4306 pVCpu->cpum.GstCtx.es.Attr.u &= ~X86DESCATTR_P;
4307
4308 pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
4309 pVCpu->cpum.GstCtx.cs.Attr.u &= ~X86DESCATTR_P;
4310
4311 pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
4312 pVCpu->cpum.GstCtx.ss.Attr.u &= ~X86DESCATTR_P;
4313
4314 pVCpu->cpum.GstCtx.ds.Sel = uNewDS;
4315 pVCpu->cpum.GstCtx.ds.Attr.u &= ~X86DESCATTR_P;
4316
4317 pVCpu->cpum.GstCtx.fs.Sel = uNewFS;
4318 pVCpu->cpum.GstCtx.fs.Attr.u &= ~X86DESCATTR_P;
4319
4320 pVCpu->cpum.GstCtx.gs.Sel = uNewGS;
4321 pVCpu->cpum.GstCtx.gs.Attr.u &= ~X86DESCATTR_P;
4322 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
4323
4324 pVCpu->cpum.GstCtx.ldtr.Sel = uNewLdt;
4325 pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_STALE;
4326 pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P;
4327 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR);
4328
4329 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
4330 {
4331 pVCpu->cpum.GstCtx.es.Attr.u |= X86DESCATTR_UNUSABLE;
4332 pVCpu->cpum.GstCtx.cs.Attr.u |= X86DESCATTR_UNUSABLE;
4333 pVCpu->cpum.GstCtx.ss.Attr.u |= X86DESCATTR_UNUSABLE;
4334 pVCpu->cpum.GstCtx.ds.Attr.u |= X86DESCATTR_UNUSABLE;
4335 pVCpu->cpum.GstCtx.fs.Attr.u |= X86DESCATTR_UNUSABLE;
4336 pVCpu->cpum.GstCtx.gs.Attr.u |= X86DESCATTR_UNUSABLE;
4337 pVCpu->cpum.GstCtx.ldtr.Attr.u |= X86DESCATTR_UNUSABLE;
4338 }
4339
4340 /*
4341 * Switch CR3 for the new task.
4342 */
4343 if ( fIsNewTSS386
4344 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG))
4345 {
4346 /** @todo Should we update and flush TLBs only if CR3 value actually changes? */
4347 int rc = CPUMSetGuestCR3(pVCpu, uNewCr3);
4348 AssertRCSuccessReturn(rc, rc);
4349
4350 /* Inform PGM. */
4351 rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
4352 AssertRCReturn(rc, rc);
4353 /* ignore informational status codes */
4354
4355 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3);
4356 }
4357
4358 /*
4359 * Switch LDTR for the new task.
4360 */
4361 if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
4362 iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt);
4363 else
4364 {
4365 Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present); /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
4366
4367 IEMSELDESC DescNewLdt;
4368 rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
4369 if (rcStrict != VINF_SUCCESS)
4370 {
4371 Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
4372 uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
4373 return rcStrict;
4374 }
4375 if ( !DescNewLdt.Legacy.Gen.u1Present
4376 || DescNewLdt.Legacy.Gen.u1DescType
4377 || DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
4378 {
4379 Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
4380 uNewLdt, DescNewLdt.Legacy.u));
4381 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
4382 }
4383
4384 pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
4385 pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
4386 pVCpu->cpum.GstCtx.ldtr.u64Base = X86DESC_BASE(&DescNewLdt.Legacy);
4387 pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
4388 pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
4389 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
4390 pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
4391 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
4392 }
4393
4394 IEMSELDESC DescSS;
4395 if (IEM_IS_V86_MODE(pVCpu))
4396 {
4397 pVCpu->iem.s.uCpl = 3;
4398 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES);
4399 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS);
4400 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS);
4401 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS);
4402 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS);
4403 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS);
4404
4405 /* Quick fix: fake DescSS. */ /** @todo fix the code further down? */
4406 DescSS.Legacy.u = 0;
4407 DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit;
4408 DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16;
4409 DescSS.Legacy.Gen.u16BaseLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base;
4410 DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16);
4411 DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24);
4412 DescSS.Legacy.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
4413 DescSS.Legacy.Gen.u2Dpl = 3;
4414 }
4415 else
4416 {
4417 uint8_t const uNewCpl = (uNewCS & X86_SEL_RPL);
4418
4419 /*
4420 * Load the stack segment for the new task.
4421 */
4422 if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
4423 {
4424 Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
4425 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4426 }
4427
4428 /* Fetch the descriptor. */
4429 rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS);
4430 if (rcStrict != VINF_SUCCESS)
4431 {
4432 Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
4433 VBOXSTRICTRC_VAL(rcStrict)));
4434 return rcStrict;
4435 }
4436
4437 /* SS must be a data segment and writable. */
4438 if ( !DescSS.Legacy.Gen.u1DescType
4439 || (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
4440 || !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
4441 {
4442 Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
4443 uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
4444 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4445 }
4446
4447 /* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
4448 if ( (uNewSS & X86_SEL_RPL) != uNewCpl
4449 || DescSS.Legacy.Gen.u2Dpl != uNewCpl)
4450 {
4451 Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
4452 uNewCpl));
4453 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4454 }
4455
4456 /* Is it there? */
4457 if (!DescSS.Legacy.Gen.u1Present)
4458 {
4459 Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
4460 return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4461 }
4462
4463 uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
4464 uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
4465
4466 /* Set the accessed bit before committing the result into SS. */
4467 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
4468 {
4469 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
4470 if (rcStrict != VINF_SUCCESS)
4471 return rcStrict;
4472 DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
4473 }
4474
4475 /* Commit SS. */
4476 pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
4477 pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
4478 pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
4479 pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit;
4480 pVCpu->cpum.GstCtx.ss.u64Base = u64Base;
4481 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4482 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
4483
4484 /* CPL has changed, update IEM before loading rest of segments. */
4485 pVCpu->iem.s.uCpl = uNewCpl;
4486
4487 /*
4488 * Load the data segments for the new task.
4489 */
4490 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES);
4491 if (rcStrict != VINF_SUCCESS)
4492 return rcStrict;
4493 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS);
4494 if (rcStrict != VINF_SUCCESS)
4495 return rcStrict;
4496 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS);
4497 if (rcStrict != VINF_SUCCESS)
4498 return rcStrict;
4499 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS);
4500 if (rcStrict != VINF_SUCCESS)
4501 return rcStrict;
4502
4503 /*
4504 * Load the code segment for the new task.
4505 */
4506 if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
4507 {
4508 Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
4509 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4510 }
4511
4512 /* Fetch the descriptor. */
4513 IEMSELDESC DescCS;
4514 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS);
4515 if (rcStrict != VINF_SUCCESS)
4516 {
4517 Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
4518 return rcStrict;
4519 }
4520
4521 /* CS must be a code segment. */
4522 if ( !DescCS.Legacy.Gen.u1DescType
4523 || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
4524 {
4525 Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
4526 DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
4527 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4528 }
4529
4530 /* For conforming CS, DPL must be less than or equal to the RPL. */
4531 if ( (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
4532 && DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
4533 {
4534 Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
4535 DescCS.Legacy.Gen.u2Dpl));
4536 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4537 }
4538
4539 /* For non-conforming CS, DPL must match RPL. */
4540 if ( !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
4541 && DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
4542 {
4543 Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
4544 DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
4545 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4546 }
4547
4548 /* Is it there? */
4549 if (!DescCS.Legacy.Gen.u1Present)
4550 {
4551 Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
4552 return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4553 }
4554
4555 cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
4556 u64Base = X86DESC_BASE(&DescCS.Legacy);
4557
4558 /* Set the accessed bit before committing the result into CS. */
4559 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
4560 {
4561 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
4562 if (rcStrict != VINF_SUCCESS)
4563 return rcStrict;
4564 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
4565 }
4566
4567 /* Commit CS. */
4568 pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
4569 pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS;
4570 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
4571 pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
4572 pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
4573 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4574 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
4575 }
4576
4577 /** @todo Debug trap. */
4578 if (fIsNewTSS386 && fNewDebugTrap)
4579 Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
4580
4581 /*
4582 * Construct the error code masks based on what caused this task switch.
4583 * See Intel Instruction reference for INT.
4584 */
4585 uint16_t uExt;
4586 if ( enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
4587 && ( !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
4588 || (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)))
4589 {
4590 uExt = 1;
4591 }
4592 else
4593 uExt = 0;
4594
4595 /*
4596 * Push any error code on to the new stack.
4597 */
4598 if (fFlags & IEM_XCPT_FLAGS_ERR)
4599 {
4600 Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
4601 uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
4602 uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2;
4603
4604 /* Check that there is sufficient space on the stack. */
4605 /** @todo Factor out segment limit checking for normal/expand down segments
4606 * into a separate function. */
4607 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
4608 {
4609 if ( pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS
4610 || pVCpu->cpum.GstCtx.esp < cbStackFrame)
4611 {
4612 /** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
4613 Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n",
4614 pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
4615 return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
4616 }
4617 }
4618 else
4619 {
4620 if ( pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
4621 || pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1))
4622 {
4623 Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n",
4624 pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
4625 return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
4626 }
4627 }
4628
4629
4630 if (fIsNewTSS386)
4631 rcStrict = iemMemStackPushU32(pVCpu, uErr);
4632 else
4633 rcStrict = iemMemStackPushU16(pVCpu, uErr);
4634 if (rcStrict != VINF_SUCCESS)
4635 {
4636 Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n",
4637 fIsNewTSS386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict)));
4638 return rcStrict;
4639 }
4640 }
4641
4642 /* Check the new EIP against the new CS limit. */
4643 if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit)
4644 {
4645 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n",
4646 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit));
4647 /** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
4648 return iemRaiseGeneralProtectionFault(pVCpu, uExt);
4649 }
4650
4651 Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
4652 pVCpu->cpum.GstCtx.ss.Sel));
4653 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
4654}
4655
4656
4657/**
4658 * Implements exceptions and interrupts for protected mode.
4659 *
4660 * @returns VBox strict status code.
4661 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4662 * @param cbInstr The number of bytes to offset rIP by in the return
4663 * address.
4664 * @param u8Vector The interrupt / exception vector number.
4665 * @param fFlags The flags.
4666 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
4667 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
4668 */
4669IEM_STATIC VBOXSTRICTRC
4670iemRaiseXcptOrIntInProtMode(PVMCPUCC pVCpu,
4671 uint8_t cbInstr,
4672 uint8_t u8Vector,
4673 uint32_t fFlags,
4674 uint16_t uErr,
4675 uint64_t uCr2)
4676{
4677 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
4678
4679 /*
4680 * Read the IDT entry.
4681 */
4682 if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
4683 {
4684 Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
4685 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4686 }
4687 X86DESC Idte;
4688 VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX,
4689 pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector);
4690 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
4691 {
4692 Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
4693 return rcStrict;
4694 }
4695 Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
4696 u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
4697 Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
4698
4699 /*
4700 * Check the descriptor type, DPL and such.
4701 * ASSUMES this is done in the same order as described for call-gate calls.
4702 */
4703 if (Idte.Gate.u1DescType)
4704 {
4705 Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
4706 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4707 }
4708 bool fTaskGate = false;
4709 uint8_t f32BitGate = true;
4710 uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
4711 switch (Idte.Gate.u4Type)
4712 {
4713 case X86_SEL_TYPE_SYS_UNDEFINED:
4714 case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
4715 case X86_SEL_TYPE_SYS_LDT:
4716 case X86_SEL_TYPE_SYS_286_TSS_BUSY:
4717 case X86_SEL_TYPE_SYS_286_CALL_GATE:
4718 case X86_SEL_TYPE_SYS_UNDEFINED2:
4719 case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
4720 case X86_SEL_TYPE_SYS_UNDEFINED3:
4721 case X86_SEL_TYPE_SYS_386_TSS_BUSY:
4722 case X86_SEL_TYPE_SYS_386_CALL_GATE:
4723 case X86_SEL_TYPE_SYS_UNDEFINED4:
4724 {
4725 /** @todo check what actually happens when the type is wrong...
4726 * esp. call gates. */
4727 Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
4728 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4729 }
4730
4731 case X86_SEL_TYPE_SYS_286_INT_GATE:
4732 f32BitGate = false;
4733 RT_FALL_THRU();
4734 case X86_SEL_TYPE_SYS_386_INT_GATE:
4735 fEflToClear |= X86_EFL_IF;
4736 break;
4737
4738 case X86_SEL_TYPE_SYS_TASK_GATE:
4739 fTaskGate = true;
4740#ifndef IEM_IMPLEMENTS_TASKSWITCH
4741 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
4742#endif
4743 break;
4744
4745 case X86_SEL_TYPE_SYS_286_TRAP_GATE:
4746 f32BitGate = false;
4747 case X86_SEL_TYPE_SYS_386_TRAP_GATE:
4748 break;
4749
4750 IEM_NOT_REACHED_DEFAULT_CASE_RET();
4751 }
4752
4753 /* Check DPL against CPL if applicable. */
4754 if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
4755 {
4756 if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
4757 {
4758 Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
4759 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4760 }
4761 }
4762
4763 /* Is it there? */
4764 if (!Idte.Gate.u1Present)
4765 {
4766 Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
4767 return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4768 }
4769
4770 /* Is it a task-gate? */
4771 if (fTaskGate)
4772 {
4773 /*
4774 * Construct the error code masks based on what caused this task switch.
4775 * See Intel Instruction reference for INT.
4776 */
4777 uint16_t const uExt = ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
4778 && !(fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)) ? 0 : 1;
4779 uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
4780 RTSEL SelTSS = Idte.Gate.u16Sel;
4781
4782 /*
4783 * Fetch the TSS descriptor in the GDT.
4784 */
4785 IEMSELDESC DescTSS;
4786 rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) | uExt);
4787 if (rcStrict != VINF_SUCCESS)
4788 {
4789 Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS,
4790 VBOXSTRICTRC_VAL(rcStrict)));
4791 return rcStrict;
4792 }
4793
4794 /* The TSS descriptor must be a system segment and be available (not busy). */
4795 if ( DescTSS.Legacy.Gen.u1DescType
4796 || ( DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
4797 && DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
4798 {
4799 Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
4800 u8Vector, SelTSS, DescTSS.Legacy.au64));
4801 return iemRaiseGeneralProtectionFault(pVCpu, (SelTSS & uSelMask) | uExt);
4802 }
4803
4804 /* The TSS must be present. */
4805 if (!DescTSS.Legacy.Gen.u1Present)
4806 {
4807 Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64));
4808 return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTSS & uSelMask) | uExt);
4809 }
4810
4811 /* Do the actual task switch. */
4812 return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT,
4813 (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip,
4814 fFlags, uErr, uCr2, SelTSS, &DescTSS);
4815 }
4816
4817 /* A null CS is bad. */
4818 RTSEL NewCS = Idte.Gate.u16Sel;
4819 if (!(NewCS & X86_SEL_MASK_OFF_RPL))
4820 {
4821 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
4822 return iemRaiseGeneralProtectionFault0(pVCpu);
4823 }
4824
4825 /* Fetch the descriptor for the new CS. */
4826 IEMSELDESC DescCS;
4827 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
4828 if (rcStrict != VINF_SUCCESS)
4829 {
4830 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
4831 return rcStrict;
4832 }
4833
4834 /* Must be a code segment. */
4835 if (!DescCS.Legacy.Gen.u1DescType)
4836 {
4837 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
4838 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
4839 }
4840 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
4841 {
4842 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
4843 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
4844 }
4845
4846 /* Don't allow lowering the privilege level. */
4847 /** @todo Does the lowering of privileges apply to software interrupts
4848 * only? This has bearings on the more-privileged or
4849 * same-privilege stack behavior further down. A testcase would
4850 * be nice. */
4851 if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
4852 {
4853 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
4854 u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
4855 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
4856 }
4857
4858 /* Make sure the selector is present. */
4859 if (!DescCS.Legacy.Gen.u1Present)
4860 {
4861 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
4862 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
4863 }
4864
4865 /* Check the new EIP against the new CS limit. */
4866 uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
4867 || Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
4868 ? Idte.Gate.u16OffsetLow
4869 : Idte.Gate.u16OffsetLow | ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
4870 uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
4871 if (uNewEip > cbLimitCS)
4872 {
4873 Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
4874 u8Vector, uNewEip, cbLimitCS, NewCS));
4875 return iemRaiseGeneralProtectionFault(pVCpu, 0);
4876 }
4877 Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS));
4878
4879 /* Calc the flag image to push. */
4880 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
4881 if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
4882 fEfl &= ~X86_EFL_RF;
4883 else
4884 fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
4885
4886 /* From V8086 mode only go to CPL 0. */
4887 uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
4888 ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
4889 if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
4890 {
4891 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
4892 return iemRaiseGeneralProtectionFault(pVCpu, 0);
4893 }
4894
4895 /*
4896 * If the privilege level changes, we need to get a new stack from the TSS.
4897 * This in turns means validating the new SS and ESP...
4898 */
4899 if (uNewCpl != pVCpu->iem.s.uCpl)
4900 {
4901 RTSEL NewSS;
4902 uint32_t uNewEsp;
4903 rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp);
4904 if (rcStrict != VINF_SUCCESS)
4905 return rcStrict;
4906
4907 IEMSELDESC DescSS;
4908 rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS);
4909 if (rcStrict != VINF_SUCCESS)
4910 return rcStrict;
4911 /* If the new SS is 16-bit, we are only going to use SP, not ESP. */
4912 if (!DescSS.Legacy.Gen.u1DefBig)
4913 {
4914 Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp));
4915 uNewEsp = (uint16_t)uNewEsp;
4916 }
4917
4918 Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
4919
4920 /* Check that there is sufficient space for the stack frame. */
4921 uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
4922 uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
4923 ? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
4924 : (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
4925
4926 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
4927 {
4928 if ( uNewEsp - 1 > cbLimitSS
4929 || uNewEsp < cbStackFrame)
4930 {
4931 Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
4932 u8Vector, NewSS, uNewEsp, cbStackFrame));
4933 return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
4934 }
4935 }
4936 else
4937 {
4938 if ( uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX)
4939 || uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1))
4940 {
4941 Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n",
4942 u8Vector, NewSS, uNewEsp, cbStackFrame));
4943 return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
4944 }
4945 }
4946
4947 /*
4948 * Start making changes.
4949 */
4950
4951 /* Set the new CPL so that stack accesses use it. */
4952 uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
4953 pVCpu->iem.s.uCpl = uNewCpl;
4954
4955 /* Create the stack frame. */
4956 RTPTRUNION uStackFrame;
4957 rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
4958 uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy), IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
4959 if (rcStrict != VINF_SUCCESS)
4960 return rcStrict;
4961 void * const pvStackFrame = uStackFrame.pv;
4962 if (f32BitGate)
4963 {
4964 if (fFlags & IEM_XCPT_FLAGS_ERR)
4965 *uStackFrame.pu32++ = uErr;
4966 uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
4967 uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
4968 uStackFrame.pu32[2] = fEfl;
4969 uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp;
4970 uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel;
4971 Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
4972 if (fEfl & X86_EFL_VM)
4973 {
4974 uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel;
4975 uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel;
4976 uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel;
4977 uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel;
4978 uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel;
4979 }
4980 }
4981 else
4982 {
4983 if (fFlags & IEM_XCPT_FLAGS_ERR)
4984 *uStackFrame.pu16++ = uErr;
4985 uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
4986 uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
4987 uStackFrame.pu16[2] = fEfl;
4988 uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp;
4989 uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel;
4990 Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp));
4991 if (fEfl & X86_EFL_VM)
4992 {
4993 uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
4994 uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel;
4995 uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel;
4996 uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel;
4997 uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel;
4998 }
4999 }
5000 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
5001 if (rcStrict != VINF_SUCCESS)
5002 return rcStrict;
5003
5004 /* Mark the selectors 'accessed' (hope this is the correct time). */
5005 /** @todo testcase: excatly _when_ are the accessed bits set - before or
5006 * after pushing the stack frame? (Write protect the gdt + stack to
5007 * find out.) */
5008 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5009 {
5010 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
5011 if (rcStrict != VINF_SUCCESS)
5012 return rcStrict;
5013 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5014 }
5015
5016 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5017 {
5018 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS);
5019 if (rcStrict != VINF_SUCCESS)
5020 return rcStrict;
5021 DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5022 }
5023
5024 /*
5025 * Start comitting the register changes (joins with the DPL=CPL branch).
5026 */
5027 pVCpu->cpum.GstCtx.ss.Sel = NewSS;
5028 pVCpu->cpum.GstCtx.ss.ValidSel = NewSS;
5029 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
5030 pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSS;
5031 pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
5032 pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
5033 /** @todo When coming from 32-bit code and operating with a 16-bit TSS and
5034 * 16-bit handler, the high word of ESP remains unchanged (i.e. only
5035 * SP is loaded).
5036 * Need to check the other combinations too:
5037 * - 16-bit TSS, 32-bit handler
5038 * - 32-bit TSS, 16-bit handler */
5039 if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
5040 pVCpu->cpum.GstCtx.sp = (uint16_t)(uNewEsp - cbStackFrame);
5041 else
5042 pVCpu->cpum.GstCtx.rsp = uNewEsp - cbStackFrame;
5043
5044 if (fEfl & X86_EFL_VM)
5045 {
5046 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs);
5047 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs);
5048 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es);
5049 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds);
5050 }
5051 }
5052 /*
5053 * Same privilege, no stack change and smaller stack frame.
5054 */
5055 else
5056 {
5057 uint64_t uNewRsp;
5058 RTPTRUNION uStackFrame;
5059 uint8_t const cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
5060 rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, &uStackFrame.pv, &uNewRsp);
5061 if (rcStrict != VINF_SUCCESS)
5062 return rcStrict;
5063 void * const pvStackFrame = uStackFrame.pv;
5064
5065 if (f32BitGate)
5066 {
5067 if (fFlags & IEM_XCPT_FLAGS_ERR)
5068 *uStackFrame.pu32++ = uErr;
5069 uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
5070 uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
5071 uStackFrame.pu32[2] = fEfl;
5072 }
5073 else
5074 {
5075 if (fFlags & IEM_XCPT_FLAGS_ERR)
5076 *uStackFrame.pu16++ = uErr;
5077 uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
5078 uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
5079 uStackFrame.pu16[2] = fEfl;
5080 }
5081 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
5082 if (rcStrict != VINF_SUCCESS)
5083 return rcStrict;
5084
5085 /* Mark the CS selector as 'accessed'. */
5086 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5087 {
5088 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
5089 if (rcStrict != VINF_SUCCESS)
5090 return rcStrict;
5091 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5092 }
5093
5094 /*
5095 * Start committing the register changes (joins with the other branch).
5096 */
5097 pVCpu->cpum.GstCtx.rsp = uNewRsp;
5098 }
5099
5100 /* ... register committing continues. */
5101 pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5102 pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5103 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
5104 pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
5105 pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
5106 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
5107
5108 pVCpu->cpum.GstCtx.rip = uNewEip; /* (The entire register is modified, see pe16_32 bs3kit tests.) */
5109 fEfl &= ~fEflToClear;
5110 IEMMISC_SET_EFL(pVCpu, fEfl);
5111
5112 if (fFlags & IEM_XCPT_FLAGS_CR2)
5113 pVCpu->cpum.GstCtx.cr2 = uCr2;
5114
5115 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
5116 iemRaiseXcptAdjustState(pVCpu, u8Vector);
5117
5118 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
5119}
5120
5121
5122/**
5123 * Implements exceptions and interrupts for long mode.
5124 *
5125 * @returns VBox strict status code.
5126 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5127 * @param cbInstr The number of bytes to offset rIP by in the return
5128 * address.
5129 * @param u8Vector The interrupt / exception vector number.
5130 * @param fFlags The flags.
5131 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
5132 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
5133 */
5134IEM_STATIC VBOXSTRICTRC
5135iemRaiseXcptOrIntInLongMode(PVMCPUCC pVCpu,
5136 uint8_t cbInstr,
5137 uint8_t u8Vector,
5138 uint32_t fFlags,
5139 uint16_t uErr,
5140 uint64_t uCr2)
5141{
5142 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
5143
5144 /*
5145 * Read the IDT entry.
5146 */
5147 uint16_t offIdt = (uint16_t)u8Vector << 4;
5148 if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7)
5149 {
5150 Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
5151 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5152 }
5153 X86DESC64 Idte;
5154#ifdef _MSC_VER /* Shut up silly compiler warning. */
5155 Idte.au64[0] = 0;
5156 Idte.au64[1] = 0;
5157#endif
5158 VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt);
5159 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
5160 rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8);
5161 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
5162 {
5163 Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
5164 return rcStrict;
5165 }
5166 Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
5167 u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
5168 Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
5169
5170 /*
5171 * Check the descriptor type, DPL and such.
5172 * ASSUMES this is done in the same order as described for call-gate calls.
5173 */
5174 if (Idte.Gate.u1DescType)
5175 {
5176 Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
5177 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5178 }
5179 uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
5180 switch (Idte.Gate.u4Type)
5181 {
5182 case AMD64_SEL_TYPE_SYS_INT_GATE:
5183 fEflToClear |= X86_EFL_IF;
5184 break;
5185 case AMD64_SEL_TYPE_SYS_TRAP_GATE:
5186 break;
5187
5188 default:
5189 Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
5190 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5191 }
5192
5193 /* Check DPL against CPL if applicable. */
5194 if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
5195 {
5196 if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
5197 {
5198 Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
5199 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5200 }
5201 }
5202
5203 /* Is it there? */
5204 if (!Idte.Gate.u1Present)
5205 {
5206 Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
5207 return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5208 }
5209
5210 /* A null CS is bad. */
5211 RTSEL NewCS = Idte.Gate.u16Sel;
5212 if (!(NewCS & X86_SEL_MASK_OFF_RPL))
5213 {
5214 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
5215 return iemRaiseGeneralProtectionFault0(pVCpu);
5216 }
5217
5218 /* Fetch the descriptor for the new CS. */
5219 IEMSELDESC DescCS;
5220 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP);
5221 if (rcStrict != VINF_SUCCESS)
5222 {
5223 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
5224 return rcStrict;
5225 }
5226
5227 /* Must be a 64-bit code segment. */
5228 if (!DescCS.Long.Gen.u1DescType)
5229 {
5230 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
5231 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
5232 }
5233 if ( !DescCS.Long.Gen.u1Long
5234 || DescCS.Long.Gen.u1DefBig
5235 || !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
5236 {
5237 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
5238 u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
5239 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
5240 }
5241
5242 /* Don't allow lowering the privilege level. For non-conforming CS
5243 selectors, the CS.DPL sets the privilege level the trap/interrupt
5244 handler runs at. For conforming CS selectors, the CPL remains
5245 unchanged, but the CS.DPL must be <= CPL. */
5246 /** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
5247 * when CPU in Ring-0. Result \#GP? */
5248 if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
5249 {
5250 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
5251 u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
5252 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
5253 }
5254
5255
5256 /* Make sure the selector is present. */
5257 if (!DescCS.Legacy.Gen.u1Present)
5258 {
5259 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
5260 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
5261 }
5262
5263 /* Check that the new RIP is canonical. */
5264 uint64_t const uNewRip = Idte.Gate.u16OffsetLow
5265 | ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
5266 | ((uint64_t)Idte.Gate.u32OffsetTop << 32);
5267 if (!IEM_IS_CANONICAL(uNewRip))
5268 {
5269 Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
5270 return iemRaiseGeneralProtectionFault0(pVCpu);
5271 }
5272
5273 /*
5274 * If the privilege level changes or if the IST isn't zero, we need to get
5275 * a new stack from the TSS.
5276 */
5277 uint64_t uNewRsp;
5278 uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
5279 ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
5280 if ( uNewCpl != pVCpu->iem.s.uCpl
5281 || Idte.Gate.u3IST != 0)
5282 {
5283 rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
5284 if (rcStrict != VINF_SUCCESS)
5285 return rcStrict;
5286 }
5287 else
5288 uNewRsp = pVCpu->cpum.GstCtx.rsp;
5289 uNewRsp &= ~(uint64_t)0xf;
5290
5291 /*
5292 * Calc the flag image to push.
5293 */
5294 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
5295 if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
5296 fEfl &= ~X86_EFL_RF;
5297 else
5298 fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
5299
5300 /*
5301 * Start making changes.
5302 */
5303 /* Set the new CPL so that stack accesses use it. */
5304 uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
5305 pVCpu->iem.s.uCpl = uNewCpl;
5306
5307 /* Create the stack frame. */
5308 uint32_t cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
5309 RTPTRUNION uStackFrame;
5310 rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
5311 uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
5312 if (rcStrict != VINF_SUCCESS)
5313 return rcStrict;
5314 void * const pvStackFrame = uStackFrame.pv;
5315
5316 if (fFlags & IEM_XCPT_FLAGS_ERR)
5317 *uStackFrame.pu64++ = uErr;
5318 uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip;
5319 uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl; /* CPL paranoia */
5320 uStackFrame.pu64[2] = fEfl;
5321 uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp;
5322 uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel;
5323 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
5324 if (rcStrict != VINF_SUCCESS)
5325 return rcStrict;
5326
5327 /* Mark the CS selectors 'accessed' (hope this is the correct time). */
5328 /** @todo testcase: excatly _when_ are the accessed bits set - before or
5329 * after pushing the stack frame? (Write protect the gdt + stack to
5330 * find out.) */
5331 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5332 {
5333 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
5334 if (rcStrict != VINF_SUCCESS)
5335 return rcStrict;
5336 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5337 }
5338
5339 /*
5340 * Start comitting the register changes.
5341 */
5342 /** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
5343 * hidden registers when interrupting 32-bit or 16-bit code! */
5344 if (uNewCpl != uOldCpl)
5345 {
5346 pVCpu->cpum.GstCtx.ss.Sel = 0 | uNewCpl;
5347 pVCpu->cpum.GstCtx.ss.ValidSel = 0 | uNewCpl;
5348 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
5349 pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
5350 pVCpu->cpum.GstCtx.ss.u64Base = 0;
5351 pVCpu->cpum.GstCtx.ss.Attr.u = (uNewCpl << X86DESCATTR_DPL_SHIFT) | X86DESCATTR_UNUSABLE;
5352 }
5353 pVCpu->cpum.GstCtx.rsp = uNewRsp - cbStackFrame;
5354 pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5355 pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5356 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
5357 pVCpu->cpum.GstCtx.cs.u32Limit = X86DESC_LIMIT_G(&DescCS.Legacy);
5358 pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
5359 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
5360 pVCpu->cpum.GstCtx.rip = uNewRip;
5361
5362 fEfl &= ~fEflToClear;
5363 IEMMISC_SET_EFL(pVCpu, fEfl);
5364
5365 if (fFlags & IEM_XCPT_FLAGS_CR2)
5366 pVCpu->cpum.GstCtx.cr2 = uCr2;
5367
5368 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
5369 iemRaiseXcptAdjustState(pVCpu, u8Vector);
5370
5371 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
5372}
5373
5374
5375/**
5376 * Implements exceptions and interrupts.
5377 *
5378 * All exceptions and interrupts goes thru this function!
5379 *
5380 * @returns VBox strict status code.
5381 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5382 * @param cbInstr The number of bytes to offset rIP by in the return
5383 * address.
5384 * @param u8Vector The interrupt / exception vector number.
5385 * @param fFlags The flags.
5386 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
5387 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
5388 */
5389DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC)
5390iemRaiseXcptOrInt(PVMCPUCC pVCpu,
5391 uint8_t cbInstr,
5392 uint8_t u8Vector,
5393 uint32_t fFlags,
5394 uint16_t uErr,
5395 uint64_t uCr2)
5396{
5397 /*
5398 * Get all the state that we might need here.
5399 */
5400 IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
5401 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
5402
5403#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */
5404 /*
5405 * Flush prefetch buffer
5406 */
5407 pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
5408#endif
5409
5410 /*
5411 * Perform the V8086 IOPL check and upgrade the fault without nesting.
5412 */
5413 if ( pVCpu->cpum.GstCtx.eflags.Bits.u1VM
5414 && pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3
5415 && (fFlags & ( IEM_XCPT_FLAGS_T_SOFT_INT
5416 | IEM_XCPT_FLAGS_BP_INSTR
5417 | IEM_XCPT_FLAGS_ICEBP_INSTR
5418 | IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
5419 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
5420 {
5421 Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
5422 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
5423 u8Vector = X86_XCPT_GP;
5424 uErr = 0;
5425 }
5426#ifdef DBGFTRACE_ENABLED
5427 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
5428 pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
5429 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp);
5430#endif
5431
5432 /*
5433 * Evaluate whether NMI blocking should be in effect.
5434 * Normally, NMI blocking is in effect whenever we inject an NMI.
5435 */
5436 bool fBlockNmi;
5437 if ( u8Vector == X86_XCPT_NMI
5438 && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
5439 fBlockNmi = true;
5440 else
5441 fBlockNmi = false;
5442
5443#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5444 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5445 {
5446 VBOXSTRICTRC rcStrict0 = iemVmxVmexitEvent(pVCpu, u8Vector, fFlags, uErr, uCr2, cbInstr);
5447 if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
5448 return rcStrict0;
5449
5450 /* If virtual-NMI blocking is in effect for the nested-guest, guest NMIs are not blocked. */
5451 if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
5452 {
5453 Assert(CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_VIRT_NMI));
5454 fBlockNmi = false;
5455 }
5456 }
5457#endif
5458
5459#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5460 if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
5461 {
5462 /*
5463 * If the event is being injected as part of VMRUN, it isn't subject to event
5464 * intercepts in the nested-guest. However, secondary exceptions that occur
5465 * during injection of any event -are- subject to exception intercepts.
5466 *
5467 * See AMD spec. 15.20 "Event Injection".
5468 */
5469 if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents)
5470 pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = true;
5471 else
5472 {
5473 /*
5474 * Check and handle if the event being raised is intercepted.
5475 */
5476 VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, u8Vector, fFlags, uErr, uCr2);
5477 if (rcStrict0 != VINF_SVM_INTERCEPT_NOT_ACTIVE)
5478 return rcStrict0;
5479 }
5480 }
5481#endif
5482
5483 /*
5484 * Set NMI blocking if necessary.
5485 */
5486 if ( fBlockNmi
5487 && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
5488 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5489
5490 /*
5491 * Do recursion accounting.
5492 */
5493 uint8_t const uPrevXcpt = pVCpu->iem.s.uCurXcpt;
5494 uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt;
5495 if (pVCpu->iem.s.cXcptRecursions == 0)
5496 Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
5497 u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2));
5498 else
5499 {
5500 Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
5501 u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt,
5502 pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt));
5503
5504 if (pVCpu->iem.s.cXcptRecursions >= 4)
5505 {
5506#ifdef DEBUG_bird
5507 AssertFailed();
5508#endif
5509 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
5510 }
5511
5512 /*
5513 * Evaluate the sequence of recurring events.
5514 */
5515 IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector,
5516 NULL /* pXcptRaiseInfo */);
5517 if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT)
5518 { /* likely */ }
5519 else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT)
5520 {
5521 Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt));
5522 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
5523 u8Vector = X86_XCPT_DF;
5524 uErr = 0;
5525#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5526 /* VMX nested-guest #DF intercept needs to be checked here. */
5527 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5528 {
5529 VBOXSTRICTRC rcStrict0 = iemVmxVmexitEventDoubleFault(pVCpu);
5530 if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
5531 return rcStrict0;
5532 }
5533#endif
5534 /* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */
5535 if (IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF))
5536 IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
5537 }
5538 else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT)
5539 {
5540 Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt));
5541 return iemInitiateCpuShutdown(pVCpu);
5542 }
5543 else if (enmRaise == IEMXCPTRAISE_CPU_HANG)
5544 {
5545 /* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */
5546 Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n"));
5547 if ( !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
5548 && !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
5549 return VERR_EM_GUEST_CPU_HANG;
5550 }
5551 else
5552 {
5553 AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n",
5554 enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags));
5555 return VERR_IEM_IPE_9;
5556 }
5557
5558 /*
5559 * The 'EXT' bit is set when an exception occurs during deliver of an external
5560 * event (such as an interrupt or earlier exception)[1]. Privileged software
5561 * exception (INT1) also sets the EXT bit[2]. Exceptions generated by software
5562 * interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set.
5563 *
5564 * [1] - Intel spec. 6.13 "Error Code"
5565 * [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection".
5566 * [3] - Intel Instruction reference for INT n.
5567 */
5568 if ( (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_T_EXT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR))
5569 && (fFlags & IEM_XCPT_FLAGS_ERR)
5570 && u8Vector != X86_XCPT_PF
5571 && u8Vector != X86_XCPT_DF)
5572 {
5573 uErr |= X86_TRAP_ERR_EXTERNAL;
5574 }
5575 }
5576
5577 pVCpu->iem.s.cXcptRecursions++;
5578 pVCpu->iem.s.uCurXcpt = u8Vector;
5579 pVCpu->iem.s.fCurXcpt = fFlags;
5580 pVCpu->iem.s.uCurXcptErr = uErr;
5581 pVCpu->iem.s.uCurXcptCr2 = uCr2;
5582
5583 /*
5584 * Extensive logging.
5585 */
5586#if defined(LOG_ENABLED) && defined(IN_RING3)
5587 if (LogIs3Enabled())
5588 {
5589 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK);
5590 PVM pVM = pVCpu->CTX_SUFF(pVM);
5591 char szRegs[4096];
5592 DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
5593 "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
5594 "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
5595 "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
5596 "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
5597 "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
5598 "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
5599 "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
5600 "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
5601 "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
5602 "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
5603 "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
5604 "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
5605 "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
5606 "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
5607 "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
5608 "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
5609 " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
5610 " efer=%016VR{efer}\n"
5611 " pat=%016VR{pat}\n"
5612 " sf_mask=%016VR{sf_mask}\n"
5613 "krnl_gs_base=%016VR{krnl_gs_base}\n"
5614 " lstar=%016VR{lstar}\n"
5615 " star=%016VR{star} cstar=%016VR{cstar}\n"
5616 "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
5617 );
5618
5619 char szInstr[256];
5620 DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
5621 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
5622 szInstr, sizeof(szInstr), NULL);
5623 Log3(("%s%s\n", szRegs, szInstr));
5624 }
5625#endif /* LOG_ENABLED */
5626
5627 /*
5628 * Call the mode specific worker function.
5629 */
5630 VBOXSTRICTRC rcStrict;
5631 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
5632 rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5633 else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
5634 rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5635 else
5636 rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5637
5638 /* Flush the prefetch buffer. */
5639#ifdef IEM_WITH_CODE_TLB
5640 pVCpu->iem.s.pbInstrBuf = NULL;
5641#else
5642 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
5643#endif
5644
5645 /*
5646 * Unwind.
5647 */
5648 pVCpu->iem.s.cXcptRecursions--;
5649 pVCpu->iem.s.uCurXcpt = uPrevXcpt;
5650 pVCpu->iem.s.fCurXcpt = fPrevXcpt;
5651 Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n",
5652 VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, pVCpu->iem.s.uCpl,
5653 pVCpu->iem.s.cXcptRecursions + 1));
5654 return rcStrict;
5655}
5656
5657#ifdef IEM_WITH_SETJMP
5658/**
5659 * See iemRaiseXcptOrInt. Will not return.
5660 */
5661IEM_STATIC DECL_NO_RETURN(void)
5662iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu,
5663 uint8_t cbInstr,
5664 uint8_t u8Vector,
5665 uint32_t fFlags,
5666 uint16_t uErr,
5667 uint64_t uCr2)
5668{
5669 VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5670 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
5671}
5672#endif
5673
5674
5675/** \#DE - 00. */
5676DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDivideError(PVMCPUCC pVCpu)
5677{
5678 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5679}
5680
5681
5682/** \#DB - 01.
5683 * @note This automatically clear DR7.GD. */
5684DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDebugException(PVMCPUCC pVCpu)
5685{
5686 /** @todo set/clear RF. */
5687 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
5688 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5689}
5690
5691
5692/** \#BR - 05. */
5693DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu)
5694{
5695 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5696}
5697
5698
5699/** \#UD - 06. */
5700DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseUndefinedOpcode(PVMCPUCC pVCpu)
5701{
5702 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5703}
5704
5705
5706/** \#NM - 07. */
5707DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu)
5708{
5709 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5710}
5711
5712
5713/** \#TS(err) - 0a. */
5714DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr)
5715{
5716 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5717}
5718
5719
5720/** \#TS(tr) - 0a. */
5721DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu)
5722{
5723 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5724 pVCpu->cpum.GstCtx.tr.Sel, 0);
5725}
5726
5727
5728/** \#TS(0) - 0a. */
5729DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu)
5730{
5731 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5732 0, 0);
5733}
5734
5735
5736/** \#TS(err) - 0a. */
5737DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel)
5738{
5739 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5740 uSel & X86_SEL_MASK_OFF_RPL, 0);
5741}
5742
5743
5744/** \#NP(err) - 0b. */
5745DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr)
5746{
5747 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5748}
5749
5750
5751/** \#NP(sel) - 0b. */
5752DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel)
5753{
5754 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5755 uSel & ~X86_SEL_RPL, 0);
5756}
5757
5758
5759/** \#SS(seg) - 0c. */
5760DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel)
5761{
5762 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5763 uSel & ~X86_SEL_RPL, 0);
5764}
5765
5766
5767/** \#SS(err) - 0c. */
5768DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr)
5769{
5770 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5771}
5772
5773
5774/** \#GP(n) - 0d. */
5775DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr)
5776{
5777 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5778}
5779
5780
5781/** \#GP(0) - 0d. */
5782DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu)
5783{
5784 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5785}
5786
5787#ifdef IEM_WITH_SETJMP
5788/** \#GP(0) - 0d. */
5789DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu)
5790{
5791 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5792}
5793#endif
5794
5795
5796/** \#GP(sel) - 0d. */
5797DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel)
5798{
5799 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5800 Sel & ~X86_SEL_RPL, 0);
5801}
5802
5803
5804/** \#GP(0) - 0d. */
5805DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseNotCanonical(PVMCPUCC pVCpu)
5806{
5807 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5808}
5809
5810
5811/** \#GP(sel) - 0d. */
5812DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess)
5813{
5814 NOREF(iSegReg); NOREF(fAccess);
5815 return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
5816 IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5817}
5818
5819#ifdef IEM_WITH_SETJMP
5820/** \#GP(sel) - 0d, longjmp. */
5821DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess)
5822{
5823 NOREF(iSegReg); NOREF(fAccess);
5824 iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
5825 IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5826}
5827#endif
5828
5829/** \#GP(sel) - 0d. */
5830DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel)
5831{
5832 NOREF(Sel);
5833 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5834}
5835
5836#ifdef IEM_WITH_SETJMP
5837/** \#GP(sel) - 0d, longjmp. */
5838DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel)
5839{
5840 NOREF(Sel);
5841 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5842}
5843#endif
5844
5845
5846/** \#GP(sel) - 0d. */
5847DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess)
5848{
5849 NOREF(iSegReg); NOREF(fAccess);
5850 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5851}
5852
5853#ifdef IEM_WITH_SETJMP
5854/** \#GP(sel) - 0d, longjmp. */
5855DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg,
5856 uint32_t fAccess)
5857{
5858 NOREF(iSegReg); NOREF(fAccess);
5859 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5860}
5861#endif
5862
5863
5864/** \#PF(n) - 0e. */
5865DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
5866{
5867 uint16_t uErr;
5868 switch (rc)
5869 {
5870 case VERR_PAGE_NOT_PRESENT:
5871 case VERR_PAGE_TABLE_NOT_PRESENT:
5872 case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
5873 case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
5874 uErr = 0;
5875 break;
5876
5877 default:
5878 AssertMsgFailed(("%Rrc\n", rc));
5879 RT_FALL_THRU();
5880 case VERR_ACCESS_DENIED:
5881 uErr = X86_TRAP_PF_P;
5882 break;
5883
5884 /** @todo reserved */
5885 }
5886
5887 if (pVCpu->iem.s.uCpl == 3)
5888 uErr |= X86_TRAP_PF_US;
5889
5890 if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
5891 && ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
5892 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) )
5893 uErr |= X86_TRAP_PF_ID;
5894
5895#if 0 /* This is so much non-sense, really. Why was it done like that? */
5896 /* Note! RW access callers reporting a WRITE protection fault, will clear
5897 the READ flag before calling. So, read-modify-write accesses (RW)
5898 can safely be reported as READ faults. */
5899 if ((fAccess & (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
5900 uErr |= X86_TRAP_PF_RW;
5901#else
5902 if (fAccess & IEM_ACCESS_TYPE_WRITE)
5903 {
5904 if (!(fAccess & IEM_ACCESS_TYPE_READ))
5905 uErr |= X86_TRAP_PF_RW;
5906 }
5907#endif
5908
5909 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR | IEM_XCPT_FLAGS_CR2,
5910 uErr, GCPtrWhere);
5911}
5912
5913#ifdef IEM_WITH_SETJMP
5914/** \#PF(n) - 0e, longjmp. */
5915IEM_STATIC DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
5916{
5917 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, fAccess, rc)));
5918}
5919#endif
5920
5921
5922/** \#MF(0) - 10. */
5923DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseMathFault(PVMCPUCC pVCpu)
5924{
5925 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5926}
5927
5928
5929/** \#AC(0) - 11. */
5930DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseAlignmentCheckException(PVMCPUCC pVCpu)
5931{
5932 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5933}
5934
5935
5936/**
5937 * Macro for calling iemCImplRaiseDivideError().
5938 *
5939 * This enables us to add/remove arguments and force different levels of
5940 * inlining as we wish.
5941 *
5942 * @return Strict VBox status code.
5943 */
5944#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
5945IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
5946{
5947 NOREF(cbInstr);
5948 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5949}
5950
5951
5952/**
5953 * Macro for calling iemCImplRaiseInvalidLockPrefix().
5954 *
5955 * This enables us to add/remove arguments and force different levels of
5956 * inlining as we wish.
5957 *
5958 * @return Strict VBox status code.
5959 */
5960#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
5961IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
5962{
5963 NOREF(cbInstr);
5964 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5965}
5966
5967
5968/**
5969 * Macro for calling iemCImplRaiseInvalidOpcode().
5970 *
5971 * This enables us to add/remove arguments and force different levels of
5972 * inlining as we wish.
5973 *
5974 * @return Strict VBox status code.
5975 */
5976#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
5977IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
5978{
5979 NOREF(cbInstr);
5980 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5981}
5982
5983
5984/** @} */
5985
5986
5987/*
5988 *
5989 * Helpers routines.
5990 * Helpers routines.
5991 * Helpers routines.
5992 *
5993 */
5994
5995/**
5996 * Recalculates the effective operand size.
5997 *
5998 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5999 */
6000IEM_STATIC void iemRecalEffOpSize(PVMCPUCC pVCpu)
6001{
6002 switch (pVCpu->iem.s.enmCpuMode)
6003 {
6004 case IEMMODE_16BIT:
6005 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
6006 break;
6007 case IEMMODE_32BIT:
6008 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
6009 break;
6010 case IEMMODE_64BIT:
6011 switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP))
6012 {
6013 case 0:
6014 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
6015 break;
6016 case IEM_OP_PRF_SIZE_OP:
6017 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
6018 break;
6019 case IEM_OP_PRF_SIZE_REX_W:
6020 case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP:
6021 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
6022 break;
6023 }
6024 break;
6025 default:
6026 AssertFailed();
6027 }
6028}
6029
6030
6031/**
6032 * Sets the default operand size to 64-bit and recalculates the effective
6033 * operand size.
6034 *
6035 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6036 */
6037IEM_STATIC void iemRecalEffOpSize64Default(PVMCPUCC pVCpu)
6038{
6039 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6040 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
6041 if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
6042 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
6043 else
6044 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
6045}
6046
6047
6048/*
6049 *
6050 * Common opcode decoders.
6051 * Common opcode decoders.
6052 * Common opcode decoders.
6053 *
6054 */
6055//#include <iprt/mem.h>
6056
6057/**
6058 * Used to add extra details about a stub case.
6059 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6060 */
6061IEM_STATIC void iemOpStubMsg2(PVMCPUCC pVCpu)
6062{
6063#if defined(LOG_ENABLED) && defined(IN_RING3)
6064 PVM pVM = pVCpu->CTX_SUFF(pVM);
6065 char szRegs[4096];
6066 DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
6067 "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
6068 "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
6069 "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
6070 "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
6071 "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
6072 "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
6073 "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
6074 "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
6075 "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
6076 "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
6077 "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
6078 "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
6079 "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
6080 "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
6081 "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
6082 "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
6083 " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
6084 " efer=%016VR{efer}\n"
6085 " pat=%016VR{pat}\n"
6086 " sf_mask=%016VR{sf_mask}\n"
6087 "krnl_gs_base=%016VR{krnl_gs_base}\n"
6088 " lstar=%016VR{lstar}\n"
6089 " star=%016VR{star} cstar=%016VR{cstar}\n"
6090 "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
6091 );
6092
6093 char szInstr[256];
6094 DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
6095 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
6096 szInstr, sizeof(szInstr), NULL);
6097
6098 RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
6099#else
6100 RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip);
6101#endif
6102}
6103
6104/**
6105 * Complains about a stub.
6106 *
6107 * Providing two versions of this macro, one for daily use and one for use when
6108 * working on IEM.
6109 */
6110#if 0
6111# define IEMOP_BITCH_ABOUT_STUB() \
6112 do { \
6113 RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
6114 iemOpStubMsg2(pVCpu); \
6115 RTAssertPanic(); \
6116 } while (0)
6117#else
6118# define IEMOP_BITCH_ABOUT_STUB() Log(("Stub: %s (line %d)\n", __FUNCTION__, __LINE__));
6119#endif
6120
6121/** Stubs an opcode. */
6122#define FNIEMOP_STUB(a_Name) \
6123 FNIEMOP_DEF(a_Name) \
6124 { \
6125 RT_NOREF_PV(pVCpu); \
6126 IEMOP_BITCH_ABOUT_STUB(); \
6127 return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
6128 } \
6129 typedef int ignore_semicolon
6130
6131/** Stubs an opcode. */
6132#define FNIEMOP_STUB_1(a_Name, a_Type0, a_Name0) \
6133 FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
6134 { \
6135 RT_NOREF_PV(pVCpu); \
6136 RT_NOREF_PV(a_Name0); \
6137 IEMOP_BITCH_ABOUT_STUB(); \
6138 return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
6139 } \
6140 typedef int ignore_semicolon
6141
6142/** Stubs an opcode which currently should raise \#UD. */
6143#define FNIEMOP_UD_STUB(a_Name) \
6144 FNIEMOP_DEF(a_Name) \
6145 { \
6146 Log(("Unsupported instruction %Rfn\n", __FUNCTION__)); \
6147 return IEMOP_RAISE_INVALID_OPCODE(); \
6148 } \
6149 typedef int ignore_semicolon
6150
6151/** Stubs an opcode which currently should raise \#UD. */
6152#define FNIEMOP_UD_STUB_1(a_Name, a_Type0, a_Name0) \
6153 FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
6154 { \
6155 RT_NOREF_PV(pVCpu); \
6156 RT_NOREF_PV(a_Name0); \
6157 Log(("Unsupported instruction %Rfn\n", __FUNCTION__)); \
6158 return IEMOP_RAISE_INVALID_OPCODE(); \
6159 } \
6160 typedef int ignore_semicolon
6161
6162
6163
6164/** @name Register Access.
6165 * @{
6166 */
6167
6168/**
6169 * Gets a reference (pointer) to the specified hidden segment register.
6170 *
6171 * @returns Hidden register reference.
6172 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6173 * @param iSegReg The segment register.
6174 */
6175IEM_STATIC PCPUMSELREG iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg)
6176{
6177 Assert(iSegReg < X86_SREG_COUNT);
6178 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6179 PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6180
6181 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
6182 return pSReg;
6183}
6184
6185
6186/**
6187 * Ensures that the given hidden segment register is up to date.
6188 *
6189 * @returns Hidden register reference.
6190 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6191 * @param pSReg The segment register.
6192 */
6193IEM_STATIC PCPUMSELREG iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
6194{
6195 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
6196 NOREF(pVCpu);
6197 return pSReg;
6198}
6199
6200
6201/**
6202 * Gets a reference (pointer) to the specified segment register (the selector
6203 * value).
6204 *
6205 * @returns Pointer to the selector variable.
6206 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6207 * @param iSegReg The segment register.
6208 */
6209DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg)
6210{
6211 Assert(iSegReg < X86_SREG_COUNT);
6212 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6213 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
6214}
6215
6216
6217/**
6218 * Fetches the selector value of a segment register.
6219 *
6220 * @returns The selector value.
6221 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6222 * @param iSegReg The segment register.
6223 */
6224DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg)
6225{
6226 Assert(iSegReg < X86_SREG_COUNT);
6227 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6228 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
6229}
6230
6231
6232/**
6233 * Fetches the base address value of a segment register.
6234 *
6235 * @returns The selector value.
6236 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6237 * @param iSegReg The segment register.
6238 */
6239DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg)
6240{
6241 Assert(iSegReg < X86_SREG_COUNT);
6242 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6243 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6244}
6245
6246
6247/**
6248 * Gets a reference (pointer) to the specified general purpose register.
6249 *
6250 * @returns Register reference.
6251 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6252 * @param iReg The general purpose register.
6253 */
6254DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg)
6255{
6256 Assert(iReg < 16);
6257 return &pVCpu->cpum.GstCtx.aGRegs[iReg];
6258}
6259
6260
6261/**
6262 * Gets a reference (pointer) to the specified 8-bit general purpose register.
6263 *
6264 * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
6265 *
6266 * @returns Register reference.
6267 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6268 * @param iReg The register.
6269 */
6270DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg)
6271{
6272 if (iReg < 4 || (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
6273 {
6274 Assert(iReg < 16);
6275 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
6276 }
6277 /* high 8-bit register. */
6278 Assert(iReg < 8);
6279 return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
6280}
6281
6282
6283/**
6284 * Gets a reference (pointer) to the specified 16-bit general purpose register.
6285 *
6286 * @returns Register reference.
6287 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6288 * @param iReg The register.
6289 */
6290DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg)
6291{
6292 Assert(iReg < 16);
6293 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
6294}
6295
6296
6297/**
6298 * Gets a reference (pointer) to the specified 32-bit general purpose register.
6299 *
6300 * @returns Register reference.
6301 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6302 * @param iReg The register.
6303 */
6304DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg)
6305{
6306 Assert(iReg < 16);
6307 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
6308}
6309
6310
6311/**
6312 * Gets a reference (pointer) to the specified 64-bit general purpose register.
6313 *
6314 * @returns Register reference.
6315 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6316 * @param iReg The register.
6317 */
6318DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg)
6319{
6320 Assert(iReg < 64);
6321 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
6322}
6323
6324
6325/**
6326 * Gets a reference (pointer) to the specified segment register's base address.
6327 *
6328 * @returns Segment register base address reference.
6329 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6330 * @param iSegReg The segment selector.
6331 */
6332DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg)
6333{
6334 Assert(iSegReg < X86_SREG_COUNT);
6335 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6336 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6337}
6338
6339
6340/**
6341 * Fetches the value of a 8-bit general purpose register.
6342 *
6343 * @returns The register value.
6344 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6345 * @param iReg The register.
6346 */
6347DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg)
6348{
6349 return *iemGRegRefU8(pVCpu, iReg);
6350}
6351
6352
6353/**
6354 * Fetches the value of a 16-bit general purpose register.
6355 *
6356 * @returns The register value.
6357 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6358 * @param iReg The register.
6359 */
6360DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg)
6361{
6362 Assert(iReg < 16);
6363 return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
6364}
6365
6366
6367/**
6368 * Fetches the value of a 32-bit general purpose register.
6369 *
6370 * @returns The register value.
6371 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6372 * @param iReg The register.
6373 */
6374DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg)
6375{
6376 Assert(iReg < 16);
6377 return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
6378}
6379
6380
6381/**
6382 * Fetches the value of a 64-bit general purpose register.
6383 *
6384 * @returns The register value.
6385 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6386 * @param iReg The register.
6387 */
6388DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg)
6389{
6390 Assert(iReg < 16);
6391 return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
6392}
6393
6394
6395/**
6396 * Adds a 8-bit signed jump offset to RIP/EIP/IP.
6397 *
6398 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6399 * segment limit.
6400 *
6401 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6402 * @param offNextInstr The offset of the next instruction.
6403 */
6404IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr)
6405{
6406 switch (pVCpu->iem.s.enmEffOpSize)
6407 {
6408 case IEMMODE_16BIT:
6409 {
6410 uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6411 if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
6412 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
6413 return iemRaiseGeneralProtectionFault0(pVCpu);
6414 pVCpu->cpum.GstCtx.rip = uNewIp;
6415 break;
6416 }
6417
6418 case IEMMODE_32BIT:
6419 {
6420 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
6421 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
6422
6423 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6424 if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
6425 return iemRaiseGeneralProtectionFault0(pVCpu);
6426 pVCpu->cpum.GstCtx.rip = uNewEip;
6427 break;
6428 }
6429
6430 case IEMMODE_64BIT:
6431 {
6432 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6433
6434 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6435 if (!IEM_IS_CANONICAL(uNewRip))
6436 return iemRaiseGeneralProtectionFault0(pVCpu);
6437 pVCpu->cpum.GstCtx.rip = uNewRip;
6438 break;
6439 }
6440
6441 IEM_NOT_REACHED_DEFAULT_CASE_RET();
6442 }
6443
6444 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6445
6446#ifndef IEM_WITH_CODE_TLB
6447 /* Flush the prefetch buffer. */
6448 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6449#endif
6450
6451 return VINF_SUCCESS;
6452}
6453
6454
6455/**
6456 * Adds a 16-bit signed jump offset to RIP/EIP/IP.
6457 *
6458 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6459 * segment limit.
6460 *
6461 * @returns Strict VBox status code.
6462 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6463 * @param offNextInstr The offset of the next instruction.
6464 */
6465IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr)
6466{
6467 Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT);
6468
6469 uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6470 if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
6471 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
6472 return iemRaiseGeneralProtectionFault0(pVCpu);
6473 /** @todo Test 16-bit jump in 64-bit mode. possible? */
6474 pVCpu->cpum.GstCtx.rip = uNewIp;
6475 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6476
6477#ifndef IEM_WITH_CODE_TLB
6478 /* Flush the prefetch buffer. */
6479 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6480#endif
6481
6482 return VINF_SUCCESS;
6483}
6484
6485
6486/**
6487 * Adds a 32-bit signed jump offset to RIP/EIP/IP.
6488 *
6489 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6490 * segment limit.
6491 *
6492 * @returns Strict VBox status code.
6493 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6494 * @param offNextInstr The offset of the next instruction.
6495 */
6496IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr)
6497{
6498 Assert(pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT);
6499
6500 if (pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT)
6501 {
6502 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
6503
6504 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6505 if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
6506 return iemRaiseGeneralProtectionFault0(pVCpu);
6507 pVCpu->cpum.GstCtx.rip = uNewEip;
6508 }
6509 else
6510 {
6511 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6512
6513 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6514 if (!IEM_IS_CANONICAL(uNewRip))
6515 return iemRaiseGeneralProtectionFault0(pVCpu);
6516 pVCpu->cpum.GstCtx.rip = uNewRip;
6517 }
6518 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6519
6520#ifndef IEM_WITH_CODE_TLB
6521 /* Flush the prefetch buffer. */
6522 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6523#endif
6524
6525 return VINF_SUCCESS;
6526}
6527
6528
6529/**
6530 * Performs a near jump to the specified address.
6531 *
6532 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6533 * segment limit.
6534 *
6535 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6536 * @param uNewRip The new RIP value.
6537 */
6538IEM_STATIC VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip)
6539{
6540 switch (pVCpu->iem.s.enmEffOpSize)
6541 {
6542 case IEMMODE_16BIT:
6543 {
6544 Assert(uNewRip <= UINT16_MAX);
6545 if ( uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit
6546 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
6547 return iemRaiseGeneralProtectionFault0(pVCpu);
6548 /** @todo Test 16-bit jump in 64-bit mode. */
6549 pVCpu->cpum.GstCtx.rip = uNewRip;
6550 break;
6551 }
6552
6553 case IEMMODE_32BIT:
6554 {
6555 Assert(uNewRip <= UINT32_MAX);
6556 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
6557 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
6558
6559 if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
6560 return iemRaiseGeneralProtectionFault0(pVCpu);
6561 pVCpu->cpum.GstCtx.rip = uNewRip;
6562 break;
6563 }
6564
6565 case IEMMODE_64BIT:
6566 {
6567 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6568
6569 if (!IEM_IS_CANONICAL(uNewRip))
6570 return iemRaiseGeneralProtectionFault0(pVCpu);
6571 pVCpu->cpum.GstCtx.rip = uNewRip;
6572 break;
6573 }
6574
6575 IEM_NOT_REACHED_DEFAULT_CASE_RET();
6576 }
6577
6578 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6579
6580#ifndef IEM_WITH_CODE_TLB
6581 /* Flush the prefetch buffer. */
6582 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6583#endif
6584
6585 return VINF_SUCCESS;
6586}
6587
6588
6589/**
6590 * Get the address of the top of the stack.
6591 *
6592 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6593 */
6594DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu)
6595{
6596 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6597 return pVCpu->cpum.GstCtx.rsp;
6598 if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6599 return pVCpu->cpum.GstCtx.esp;
6600 return pVCpu->cpum.GstCtx.sp;
6601}
6602
6603
6604/**
6605 * Updates the RIP/EIP/IP to point to the next instruction.
6606 *
6607 * This function leaves the EFLAGS.RF flag alone.
6608 *
6609 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6610 * @param cbInstr The number of bytes to add.
6611 */
6612IEM_STATIC void iemRegAddToRipKeepRF(PVMCPUCC pVCpu, uint8_t cbInstr)
6613{
6614 switch (pVCpu->iem.s.enmCpuMode)
6615 {
6616 case IEMMODE_16BIT:
6617 Assert(pVCpu->cpum.GstCtx.rip <= UINT16_MAX);
6618 pVCpu->cpum.GstCtx.eip += cbInstr;
6619 pVCpu->cpum.GstCtx.eip &= UINT32_C(0xffff);
6620 break;
6621
6622 case IEMMODE_32BIT:
6623 pVCpu->cpum.GstCtx.eip += cbInstr;
6624 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
6625 break;
6626
6627 case IEMMODE_64BIT:
6628 pVCpu->cpum.GstCtx.rip += cbInstr;
6629 break;
6630 default: AssertFailed();
6631 }
6632}
6633
6634
6635#if 0
6636/**
6637 * Updates the RIP/EIP/IP to point to the next instruction.
6638 *
6639 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6640 */
6641IEM_STATIC void iemRegUpdateRipKeepRF(PVMCPUCC pVCpu)
6642{
6643 return iemRegAddToRipKeepRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
6644}
6645#endif
6646
6647
6648
6649/**
6650 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
6651 *
6652 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6653 * @param cbInstr The number of bytes to add.
6654 */
6655IEM_STATIC void iemRegAddToRipAndClearRF(PVMCPUCC pVCpu, uint8_t cbInstr)
6656{
6657 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6658
6659 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
6660#if ARCH_BITS >= 64
6661 static uint64_t const s_aRipMasks[] = { UINT64_C(0xffffffff), UINT64_C(0xffffffff), UINT64_MAX };
6662 Assert(pVCpu->cpum.GstCtx.rip <= s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode]);
6663 pVCpu->cpum.GstCtx.rip = (pVCpu->cpum.GstCtx.rip + cbInstr) & s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode];
6664#else
6665 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6666 pVCpu->cpum.GstCtx.rip += cbInstr;
6667 else
6668 pVCpu->cpum.GstCtx.eip += cbInstr;
6669#endif
6670}
6671
6672
6673/**
6674 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
6675 *
6676 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6677 */
6678IEM_STATIC void iemRegUpdateRipAndClearRF(PVMCPUCC pVCpu)
6679{
6680 return iemRegAddToRipAndClearRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
6681}
6682
6683
6684/**
6685 * Adds to the stack pointer.
6686 *
6687 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6688 * @param cbToAdd The number of bytes to add (8-bit!).
6689 */
6690DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd)
6691{
6692 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6693 pVCpu->cpum.GstCtx.rsp += cbToAdd;
6694 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6695 pVCpu->cpum.GstCtx.esp += cbToAdd;
6696 else
6697 pVCpu->cpum.GstCtx.sp += cbToAdd;
6698}
6699
6700
6701/**
6702 * Subtracts from the stack pointer.
6703 *
6704 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6705 * @param cbToSub The number of bytes to subtract (8-bit!).
6706 */
6707DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub)
6708{
6709 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6710 pVCpu->cpum.GstCtx.rsp -= cbToSub;
6711 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6712 pVCpu->cpum.GstCtx.esp -= cbToSub;
6713 else
6714 pVCpu->cpum.GstCtx.sp -= cbToSub;
6715}
6716
6717
6718/**
6719 * Adds to the temporary stack pointer.
6720 *
6721 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6722 * @param pTmpRsp The temporary SP/ESP/RSP to update.
6723 * @param cbToAdd The number of bytes to add (16-bit).
6724 */
6725DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd)
6726{
6727 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6728 pTmpRsp->u += cbToAdd;
6729 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6730 pTmpRsp->DWords.dw0 += cbToAdd;
6731 else
6732 pTmpRsp->Words.w0 += cbToAdd;
6733}
6734
6735
6736/**
6737 * Subtracts from the temporary stack pointer.
6738 *
6739 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6740 * @param pTmpRsp The temporary SP/ESP/RSP to update.
6741 * @param cbToSub The number of bytes to subtract.
6742 * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is
6743 * expecting that.
6744 */
6745DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub)
6746{
6747 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6748 pTmpRsp->u -= cbToSub;
6749 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6750 pTmpRsp->DWords.dw0 -= cbToSub;
6751 else
6752 pTmpRsp->Words.w0 -= cbToSub;
6753}
6754
6755
6756/**
6757 * Calculates the effective stack address for a push of the specified size as
6758 * well as the new RSP value (upper bits may be masked).
6759 *
6760 * @returns Effective stack addressf for the push.
6761 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6762 * @param cbItem The size of the stack item to pop.
6763 * @param puNewRsp Where to return the new RSP value.
6764 */
6765DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
6766{
6767 RTUINT64U uTmpRsp;
6768 RTGCPTR GCPtrTop;
6769 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
6770
6771 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6772 GCPtrTop = uTmpRsp.u -= cbItem;
6773 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6774 GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
6775 else
6776 GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
6777 *puNewRsp = uTmpRsp.u;
6778 return GCPtrTop;
6779}
6780
6781
6782/**
6783 * Gets the current stack pointer and calculates the value after a pop of the
6784 * specified size.
6785 *
6786 * @returns Current stack pointer.
6787 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6788 * @param cbItem The size of the stack item to pop.
6789 * @param puNewRsp Where to return the new RSP value.
6790 */
6791DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
6792{
6793 RTUINT64U uTmpRsp;
6794 RTGCPTR GCPtrTop;
6795 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
6796
6797 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6798 {
6799 GCPtrTop = uTmpRsp.u;
6800 uTmpRsp.u += cbItem;
6801 }
6802 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6803 {
6804 GCPtrTop = uTmpRsp.DWords.dw0;
6805 uTmpRsp.DWords.dw0 += cbItem;
6806 }
6807 else
6808 {
6809 GCPtrTop = uTmpRsp.Words.w0;
6810 uTmpRsp.Words.w0 += cbItem;
6811 }
6812 *puNewRsp = uTmpRsp.u;
6813 return GCPtrTop;
6814}
6815
6816
6817/**
6818 * Calculates the effective stack address for a push of the specified size as
6819 * well as the new temporary RSP value (upper bits may be masked).
6820 *
6821 * @returns Effective stack addressf for the push.
6822 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6823 * @param pTmpRsp The temporary stack pointer. This is updated.
6824 * @param cbItem The size of the stack item to pop.
6825 */
6826DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
6827{
6828 RTGCPTR GCPtrTop;
6829
6830 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6831 GCPtrTop = pTmpRsp->u -= cbItem;
6832 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6833 GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
6834 else
6835 GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
6836 return GCPtrTop;
6837}
6838
6839
6840/**
6841 * Gets the effective stack address for a pop of the specified size and
6842 * calculates and updates the temporary RSP.
6843 *
6844 * @returns Current stack pointer.
6845 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6846 * @param pTmpRsp The temporary stack pointer. This is updated.
6847 * @param cbItem The size of the stack item to pop.
6848 */
6849DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
6850{
6851 RTGCPTR GCPtrTop;
6852 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6853 {
6854 GCPtrTop = pTmpRsp->u;
6855 pTmpRsp->u += cbItem;
6856 }
6857 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6858 {
6859 GCPtrTop = pTmpRsp->DWords.dw0;
6860 pTmpRsp->DWords.dw0 += cbItem;
6861 }
6862 else
6863 {
6864 GCPtrTop = pTmpRsp->Words.w0;
6865 pTmpRsp->Words.w0 += cbItem;
6866 }
6867 return GCPtrTop;
6868}
6869
6870/** @} */
6871
6872
6873/** @name FPU access and helpers.
6874 *
6875 * @{
6876 */
6877
6878
6879/**
6880 * Hook for preparing to use the host FPU.
6881 *
6882 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6883 *
6884 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6885 */
6886DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu)
6887{
6888#ifdef IN_RING3
6889 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
6890#else
6891 CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
6892#endif
6893 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6894}
6895
6896
6897/**
6898 * Hook for preparing to use the host FPU for SSE.
6899 *
6900 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6901 *
6902 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6903 */
6904DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu)
6905{
6906 iemFpuPrepareUsage(pVCpu);
6907}
6908
6909
6910/**
6911 * Hook for preparing to use the host FPU for AVX.
6912 *
6913 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6914 *
6915 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6916 */
6917DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu)
6918{
6919 iemFpuPrepareUsage(pVCpu);
6920}
6921
6922
6923/**
6924 * Hook for actualizing the guest FPU state before the interpreter reads it.
6925 *
6926 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6927 *
6928 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6929 */
6930DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu)
6931{
6932#ifdef IN_RING3
6933 NOREF(pVCpu);
6934#else
6935 CPUMRZFpuStateActualizeForRead(pVCpu);
6936#endif
6937 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6938}
6939
6940
6941/**
6942 * Hook for actualizing the guest FPU state before the interpreter changes it.
6943 *
6944 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6945 *
6946 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6947 */
6948DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu)
6949{
6950#ifdef IN_RING3
6951 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
6952#else
6953 CPUMRZFpuStateActualizeForChange(pVCpu);
6954#endif
6955 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6956}
6957
6958
6959/**
6960 * Hook for actualizing the guest XMM0..15 and MXCSR register state for read
6961 * only.
6962 *
6963 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6964 *
6965 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6966 */
6967DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu)
6968{
6969#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
6970 NOREF(pVCpu);
6971#else
6972 CPUMRZFpuStateActualizeSseForRead(pVCpu);
6973#endif
6974 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6975}
6976
6977
6978/**
6979 * Hook for actualizing the guest XMM0..15 and MXCSR register state for
6980 * read+write.
6981 *
6982 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6983 *
6984 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6985 */
6986DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu)
6987{
6988#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
6989 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
6990#else
6991 CPUMRZFpuStateActualizeForChange(pVCpu);
6992#endif
6993 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6994}
6995
6996
6997/**
6998 * Hook for actualizing the guest YMM0..15 and MXCSR register state for read
6999 * only.
7000 *
7001 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
7002 *
7003 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7004 */
7005DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu)
7006{
7007#ifdef IN_RING3
7008 NOREF(pVCpu);
7009#else
7010 CPUMRZFpuStateActualizeAvxForRead(pVCpu);
7011#endif
7012 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
7013}
7014
7015
7016/**
7017 * Hook for actualizing the guest YMM0..15 and MXCSR register state for
7018 * read+write.
7019 *
7020 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
7021 *
7022 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7023 */
7024DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu)
7025{
7026#ifdef IN_RING3
7027 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
7028#else
7029 CPUMRZFpuStateActualizeForChange(pVCpu);
7030#endif
7031 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
7032}
7033
7034
7035/**
7036 * Stores a QNaN value into a FPU register.
7037 *
7038 * @param pReg Pointer to the register.
7039 */
7040DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
7041{
7042 pReg->au32[0] = UINT32_C(0x00000000);
7043 pReg->au32[1] = UINT32_C(0xc0000000);
7044 pReg->au16[4] = UINT16_C(0xffff);
7045}
7046
7047
7048/**
7049 * Updates the FOP, FPU.CS and FPUIP registers.
7050 *
7051 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7052 * @param pFpuCtx The FPU context.
7053 */
7054DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx)
7055{
7056 Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
7057 pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
7058 /** @todo x87.CS and FPUIP needs to be kept seperately. */
7059 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
7060 {
7061 /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
7062 * happens in real mode here based on the fnsave and fnstenv images. */
7063 pFpuCtx->CS = 0;
7064 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
7065 }
7066 else
7067 {
7068 pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
7069 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
7070 }
7071}
7072
7073
7074/**
7075 * Updates the x87.DS and FPUDP registers.
7076 *
7077 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7078 * @param pFpuCtx The FPU context.
7079 * @param iEffSeg The effective segment register.
7080 * @param GCPtrEff The effective address relative to @a iEffSeg.
7081 */
7082DECLINLINE(void) iemFpuUpdateDP(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7083{
7084 RTSEL sel;
7085 switch (iEffSeg)
7086 {
7087 case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break;
7088 case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break;
7089 case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break;
7090 case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break;
7091 case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break;
7092 case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break;
7093 default:
7094 AssertMsgFailed(("%d\n", iEffSeg));
7095 sel = pVCpu->cpum.GstCtx.ds.Sel;
7096 }
7097 /** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */
7098 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
7099 {
7100 pFpuCtx->DS = 0;
7101 pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4);
7102 }
7103 else
7104 {
7105 pFpuCtx->DS = sel;
7106 pFpuCtx->FPUDP = GCPtrEff;
7107 }
7108}
7109
7110
7111/**
7112 * Rotates the stack registers in the push direction.
7113 *
7114 * @param pFpuCtx The FPU context.
7115 * @remarks This is a complete waste of time, but fxsave stores the registers in
7116 * stack order.
7117 */
7118DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx)
7119{
7120 RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80;
7121 pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80;
7122 pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80;
7123 pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80;
7124 pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80;
7125 pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80;
7126 pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80;
7127 pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80;
7128 pFpuCtx->aRegs[0].r80 = r80Tmp;
7129}
7130
7131
7132/**
7133 * Rotates the stack registers in the pop direction.
7134 *
7135 * @param pFpuCtx The FPU context.
7136 * @remarks This is a complete waste of time, but fxsave stores the registers in
7137 * stack order.
7138 */
7139DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx)
7140{
7141 RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80;
7142 pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80;
7143 pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80;
7144 pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80;
7145 pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80;
7146 pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80;
7147 pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80;
7148 pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80;
7149 pFpuCtx->aRegs[7].r80 = r80Tmp;
7150}
7151
7152
7153/**
7154 * Updates FSW and pushes a FPU result onto the FPU stack if no pending
7155 * exception prevents it.
7156 *
7157 * @param pResult The FPU operation result to push.
7158 * @param pFpuCtx The FPU context.
7159 */
7160IEM_STATIC void iemFpuMaybePushResult(PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx)
7161{
7162 /* Update FSW and bail if there are pending exceptions afterwards. */
7163 uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
7164 fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
7165 if ( (fFsw & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
7166 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
7167 {
7168 pFpuCtx->FSW = fFsw;
7169 return;
7170 }
7171
7172 uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
7173 if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
7174 {
7175 /* All is fine, push the actual value. */
7176 pFpuCtx->FTW |= RT_BIT(iNewTop);
7177 pFpuCtx->aRegs[7].r80 = pResult->r80Result;
7178 }
7179 else if (pFpuCtx->FCW & X86_FCW_IM)
7180 {
7181 /* Masked stack overflow, push QNaN. */
7182 fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
7183 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7184 }
7185 else
7186 {
7187 /* Raise stack overflow, don't push anything. */
7188 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
7189 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
7190 return;
7191 }
7192
7193 fFsw &= ~X86_FSW_TOP_MASK;
7194 fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
7195 pFpuCtx->FSW = fFsw;
7196
7197 iemFpuRotateStackPush(pFpuCtx);
7198}
7199
7200
7201/**
7202 * Stores a result in a FPU register and updates the FSW and FTW.
7203 *
7204 * @param pFpuCtx The FPU context.
7205 * @param pResult The result to store.
7206 * @param iStReg Which FPU register to store it in.
7207 */
7208IEM_STATIC void iemFpuStoreResultOnly(PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg)
7209{
7210 Assert(iStReg < 8);
7211 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7212 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7213 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_TOP_MASK;
7214 pFpuCtx->FTW |= RT_BIT(iReg);
7215 pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result;
7216}
7217
7218
7219/**
7220 * Only updates the FPU status word (FSW) with the result of the current
7221 * instruction.
7222 *
7223 * @param pFpuCtx The FPU context.
7224 * @param u16FSW The FSW output of the current instruction.
7225 */
7226IEM_STATIC void iemFpuUpdateFSWOnly(PX86FXSTATE pFpuCtx, uint16_t u16FSW)
7227{
7228 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7229 pFpuCtx->FSW |= u16FSW & ~X86_FSW_TOP_MASK;
7230}
7231
7232
7233/**
7234 * Pops one item off the FPU stack if no pending exception prevents it.
7235 *
7236 * @param pFpuCtx The FPU context.
7237 */
7238IEM_STATIC void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx)
7239{
7240 /* Check pending exceptions. */
7241 uint16_t uFSW = pFpuCtx->FSW;
7242 if ( (pFpuCtx->FSW & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
7243 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
7244 return;
7245
7246 /* TOP--. */
7247 uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
7248 uFSW &= ~X86_FSW_TOP_MASK;
7249 uFSW |= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
7250 pFpuCtx->FSW = uFSW;
7251
7252 /* Mark the previous ST0 as empty. */
7253 iOldTop >>= X86_FSW_TOP_SHIFT;
7254 pFpuCtx->FTW &= ~RT_BIT(iOldTop);
7255
7256 /* Rotate the registers. */
7257 iemFpuRotateStackPop(pFpuCtx);
7258}
7259
7260
7261/**
7262 * Pushes a FPU result onto the FPU stack if no pending exception prevents it.
7263 *
7264 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7265 * @param pResult The FPU operation result to push.
7266 */
7267IEM_STATIC void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult)
7268{
7269 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7270 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7271 iemFpuMaybePushResult(pResult, pFpuCtx);
7272}
7273
7274
7275/**
7276 * Pushes a FPU result onto the FPU stack if no pending exception prevents it,
7277 * and sets FPUDP and FPUDS.
7278 *
7279 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7280 * @param pResult The FPU operation result to push.
7281 * @param iEffSeg The effective segment register.
7282 * @param GCPtrEff The effective address relative to @a iEffSeg.
7283 */
7284IEM_STATIC void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7285{
7286 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7287 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7288 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7289 iemFpuMaybePushResult(pResult, pFpuCtx);
7290}
7291
7292
7293/**
7294 * Replace ST0 with the first value and push the second onto the FPU stack,
7295 * unless a pending exception prevents it.
7296 *
7297 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7298 * @param pResult The FPU operation result to store and push.
7299 */
7300IEM_STATIC void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult)
7301{
7302 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7303 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7304
7305 /* Update FSW and bail if there are pending exceptions afterwards. */
7306 uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
7307 fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
7308 if ( (fFsw & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
7309 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
7310 {
7311 pFpuCtx->FSW = fFsw;
7312 return;
7313 }
7314
7315 uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
7316 if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
7317 {
7318 /* All is fine, push the actual value. */
7319 pFpuCtx->FTW |= RT_BIT(iNewTop);
7320 pFpuCtx->aRegs[0].r80 = pResult->r80Result1;
7321 pFpuCtx->aRegs[7].r80 = pResult->r80Result2;
7322 }
7323 else if (pFpuCtx->FCW & X86_FCW_IM)
7324 {
7325 /* Masked stack overflow, push QNaN. */
7326 fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
7327 iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
7328 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7329 }
7330 else
7331 {
7332 /* Raise stack overflow, don't push anything. */
7333 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
7334 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
7335 return;
7336 }
7337
7338 fFsw &= ~X86_FSW_TOP_MASK;
7339 fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
7340 pFpuCtx->FSW = fFsw;
7341
7342 iemFpuRotateStackPush(pFpuCtx);
7343}
7344
7345
7346/**
7347 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
7348 * FOP.
7349 *
7350 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7351 * @param pResult The result to store.
7352 * @param iStReg Which FPU register to store it in.
7353 */
7354IEM_STATIC void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg)
7355{
7356 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7357 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7358 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7359}
7360
7361
7362/**
7363 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
7364 * FOP, and then pops the stack.
7365 *
7366 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7367 * @param pResult The result to store.
7368 * @param iStReg Which FPU register to store it in.
7369 */
7370IEM_STATIC void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg)
7371{
7372 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7373 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7374 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7375 iemFpuMaybePopOne(pFpuCtx);
7376}
7377
7378
7379/**
7380 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
7381 * FPUDP, and FPUDS.
7382 *
7383 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7384 * @param pResult The result to store.
7385 * @param iStReg Which FPU register to store it in.
7386 * @param iEffSeg The effective memory operand selector register.
7387 * @param GCPtrEff The effective memory operand offset.
7388 */
7389IEM_STATIC void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
7390 uint8_t iEffSeg, RTGCPTR GCPtrEff)
7391{
7392 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7393 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7394 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7395 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7396}
7397
7398
7399/**
7400 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
7401 * FPUDP, and FPUDS, and then pops the stack.
7402 *
7403 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7404 * @param pResult The result to store.
7405 * @param iStReg Which FPU register to store it in.
7406 * @param iEffSeg The effective memory operand selector register.
7407 * @param GCPtrEff The effective memory operand offset.
7408 */
7409IEM_STATIC void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult,
7410 uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7411{
7412 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7413 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7414 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7415 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7416 iemFpuMaybePopOne(pFpuCtx);
7417}
7418
7419
7420/**
7421 * Updates the FOP, FPUIP, and FPUCS. For FNOP.
7422 *
7423 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7424 */
7425IEM_STATIC void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu)
7426{
7427 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7428 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7429}
7430
7431
7432/**
7433 * Marks the specified stack register as free (for FFREE).
7434 *
7435 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7436 * @param iStReg The register to free.
7437 */
7438IEM_STATIC void iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg)
7439{
7440 Assert(iStReg < 8);
7441 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7442 uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7443 pFpuCtx->FTW &= ~RT_BIT(iReg);
7444}
7445
7446
7447/**
7448 * Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
7449 *
7450 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7451 */
7452IEM_STATIC void iemFpuStackIncTop(PVMCPUCC pVCpu)
7453{
7454 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7455 uint16_t uFsw = pFpuCtx->FSW;
7456 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
7457 uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
7458 uFsw &= ~X86_FSW_TOP_MASK;
7459 uFsw |= uTop;
7460 pFpuCtx->FSW = uFsw;
7461}
7462
7463
7464/**
7465 * Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
7466 *
7467 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7468 */
7469IEM_STATIC void iemFpuStackDecTop(PVMCPUCC pVCpu)
7470{
7471 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7472 uint16_t uFsw = pFpuCtx->FSW;
7473 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
7474 uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
7475 uFsw &= ~X86_FSW_TOP_MASK;
7476 uFsw |= uTop;
7477 pFpuCtx->FSW = uFsw;
7478}
7479
7480
7481/**
7482 * Updates the FSW, FOP, FPUIP, and FPUCS.
7483 *
7484 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7485 * @param u16FSW The FSW from the current instruction.
7486 */
7487IEM_STATIC void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW)
7488{
7489 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7490 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7491 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7492}
7493
7494
7495/**
7496 * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
7497 *
7498 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7499 * @param u16FSW The FSW from the current instruction.
7500 */
7501IEM_STATIC void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW)
7502{
7503 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7504 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7505 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7506 iemFpuMaybePopOne(pFpuCtx);
7507}
7508
7509
7510/**
7511 * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
7512 *
7513 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7514 * @param u16FSW The FSW from the current instruction.
7515 * @param iEffSeg The effective memory operand selector register.
7516 * @param GCPtrEff The effective memory operand offset.
7517 */
7518IEM_STATIC void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7519{
7520 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7521 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7522 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7523 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7524}
7525
7526
7527/**
7528 * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
7529 *
7530 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7531 * @param u16FSW The FSW from the current instruction.
7532 */
7533IEM_STATIC void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW)
7534{
7535 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7536 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7537 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7538 iemFpuMaybePopOne(pFpuCtx);
7539 iemFpuMaybePopOne(pFpuCtx);
7540}
7541
7542
7543/**
7544 * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
7545 *
7546 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7547 * @param u16FSW The FSW from the current instruction.
7548 * @param iEffSeg The effective memory operand selector register.
7549 * @param GCPtrEff The effective memory operand offset.
7550 */
7551IEM_STATIC void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7552{
7553 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7554 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7555 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7556 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7557 iemFpuMaybePopOne(pFpuCtx);
7558}
7559
7560
7561/**
7562 * Worker routine for raising an FPU stack underflow exception.
7563 *
7564 * @param pFpuCtx The FPU context.
7565 * @param iStReg The stack register being accessed.
7566 */
7567IEM_STATIC void iemFpuStackUnderflowOnly(PX86FXSTATE pFpuCtx, uint8_t iStReg)
7568{
7569 Assert(iStReg < 8 || iStReg == UINT8_MAX);
7570 if (pFpuCtx->FCW & X86_FCW_IM)
7571 {
7572 /* Masked underflow. */
7573 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7574 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
7575 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7576 if (iStReg != UINT8_MAX)
7577 {
7578 pFpuCtx->FTW |= RT_BIT(iReg);
7579 iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
7580 }
7581 }
7582 else
7583 {
7584 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7585 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7586 }
7587}
7588
7589
7590/**
7591 * Raises a FPU stack underflow exception.
7592 *
7593 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7594 * @param iStReg The destination register that should be loaded
7595 * with QNaN if \#IS is not masked. Specify
7596 * UINT8_MAX if none (like for fcom).
7597 */
7598DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg)
7599{
7600 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7601 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7602 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7603}
7604
7605
7606DECL_NO_INLINE(IEM_STATIC, void)
7607iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7608{
7609 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7610 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7611 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7612 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7613}
7614
7615
7616DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg)
7617{
7618 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7619 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7620 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7621 iemFpuMaybePopOne(pFpuCtx);
7622}
7623
7624
7625DECL_NO_INLINE(IEM_STATIC, void)
7626iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7627{
7628 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7629 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7630 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7631 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7632 iemFpuMaybePopOne(pFpuCtx);
7633}
7634
7635
7636DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu)
7637{
7638 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7639 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7640 iemFpuStackUnderflowOnly(pFpuCtx, UINT8_MAX);
7641 iemFpuMaybePopOne(pFpuCtx);
7642 iemFpuMaybePopOne(pFpuCtx);
7643}
7644
7645
7646DECL_NO_INLINE(IEM_STATIC, void)
7647iemFpuStackPushUnderflow(PVMCPUCC pVCpu)
7648{
7649 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7650 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7651
7652 if (pFpuCtx->FCW & X86_FCW_IM)
7653 {
7654 /* Masked overflow - Push QNaN. */
7655 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
7656 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
7657 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
7658 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
7659 pFpuCtx->FTW |= RT_BIT(iNewTop);
7660 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7661 iemFpuRotateStackPush(pFpuCtx);
7662 }
7663 else
7664 {
7665 /* Exception pending - don't change TOP or the register stack. */
7666 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7667 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7668 }
7669}
7670
7671
7672DECL_NO_INLINE(IEM_STATIC, void)
7673iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu)
7674{
7675 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7676 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7677
7678 if (pFpuCtx->FCW & X86_FCW_IM)
7679 {
7680 /* Masked overflow - Push QNaN. */
7681 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
7682 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
7683 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
7684 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
7685 pFpuCtx->FTW |= RT_BIT(iNewTop);
7686 iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
7687 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7688 iemFpuRotateStackPush(pFpuCtx);
7689 }
7690 else
7691 {
7692 /* Exception pending - don't change TOP or the register stack. */
7693 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7694 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7695 }
7696}
7697
7698
7699/**
7700 * Worker routine for raising an FPU stack overflow exception on a push.
7701 *
7702 * @param pFpuCtx The FPU context.
7703 */
7704IEM_STATIC void iemFpuStackPushOverflowOnly(PX86FXSTATE pFpuCtx)
7705{
7706 if (pFpuCtx->FCW & X86_FCW_IM)
7707 {
7708 /* Masked overflow. */
7709 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
7710 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
7711 pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF;
7712 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
7713 pFpuCtx->FTW |= RT_BIT(iNewTop);
7714 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7715 iemFpuRotateStackPush(pFpuCtx);
7716 }
7717 else
7718 {
7719 /* Exception pending - don't change TOP or the register stack. */
7720 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7721 pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7722 }
7723}
7724
7725
7726/**
7727 * Raises a FPU stack overflow exception on a push.
7728 *
7729 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7730 */
7731DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackPushOverflow(PVMCPUCC pVCpu)
7732{
7733 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7734 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7735 iemFpuStackPushOverflowOnly(pFpuCtx);
7736}
7737
7738
7739/**
7740 * Raises a FPU stack overflow exception on a push with a memory operand.
7741 *
7742 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7743 * @param iEffSeg The effective memory operand selector register.
7744 * @param GCPtrEff The effective memory operand offset.
7745 */
7746DECL_NO_INLINE(IEM_STATIC, void)
7747iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7748{
7749 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7750 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7751 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7752 iemFpuStackPushOverflowOnly(pFpuCtx);
7753}
7754
7755
7756IEM_STATIC int iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg)
7757{
7758 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7759 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7760 if (pFpuCtx->FTW & RT_BIT(iReg))
7761 return VINF_SUCCESS;
7762 return VERR_NOT_FOUND;
7763}
7764
7765
7766IEM_STATIC int iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
7767{
7768 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7769 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7770 if (pFpuCtx->FTW & RT_BIT(iReg))
7771 {
7772 *ppRef = &pFpuCtx->aRegs[iStReg].r80;
7773 return VINF_SUCCESS;
7774 }
7775 return VERR_NOT_FOUND;
7776}
7777
7778
7779IEM_STATIC int iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
7780 uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
7781{
7782 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7783 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
7784 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
7785 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
7786 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
7787 {
7788 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
7789 *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
7790 return VINF_SUCCESS;
7791 }
7792 return VERR_NOT_FOUND;
7793}
7794
7795
7796IEM_STATIC int iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
7797{
7798 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7799 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
7800 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
7801 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
7802 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
7803 {
7804 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
7805 return VINF_SUCCESS;
7806 }
7807 return VERR_NOT_FOUND;
7808}
7809
7810
7811/**
7812 * Updates the FPU exception status after FCW is changed.
7813 *
7814 * @param pFpuCtx The FPU context.
7815 */
7816IEM_STATIC void iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx)
7817{
7818 uint16_t u16Fsw = pFpuCtx->FSW;
7819 if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
7820 u16Fsw |= X86_FSW_ES | X86_FSW_B;
7821 else
7822 u16Fsw &= ~(X86_FSW_ES | X86_FSW_B);
7823 pFpuCtx->FSW = u16Fsw;
7824}
7825
7826
7827/**
7828 * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
7829 *
7830 * @returns The full FTW.
7831 * @param pFpuCtx The FPU context.
7832 */
7833IEM_STATIC uint16_t iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx)
7834{
7835 uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
7836 uint16_t u16Ftw = 0;
7837 unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
7838 for (unsigned iSt = 0; iSt < 8; iSt++)
7839 {
7840 unsigned const iReg = (iSt + iTop) & 7;
7841 if (!(u8Ftw & RT_BIT(iReg)))
7842 u16Ftw |= 3 << (iReg * 2); /* empty */
7843 else
7844 {
7845 uint16_t uTag;
7846 PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
7847 if (pr80Reg->s.uExponent == 0x7fff)
7848 uTag = 2; /* Exponent is all 1's => Special. */
7849 else if (pr80Reg->s.uExponent == 0x0000)
7850 {
7851 if (pr80Reg->s.u64Mantissa == 0x0000)
7852 uTag = 1; /* All bits are zero => Zero. */
7853 else
7854 uTag = 2; /* Must be special. */
7855 }
7856 else if (pr80Reg->s.u64Mantissa & RT_BIT_64(63)) /* The J bit. */
7857 uTag = 0; /* Valid. */
7858 else
7859 uTag = 2; /* Must be special. */
7860
7861 u16Ftw |= uTag << (iReg * 2); /* empty */
7862 }
7863 }
7864
7865 return u16Ftw;
7866}
7867
7868
7869/**
7870 * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
7871 *
7872 * @returns The compressed FTW.
7873 * @param u16FullFtw The full FTW to convert.
7874 */
7875IEM_STATIC uint16_t iemFpuCompressFtw(uint16_t u16FullFtw)
7876{
7877 uint8_t u8Ftw = 0;
7878 for (unsigned i = 0; i < 8; i++)
7879 {
7880 if ((u16FullFtw & 3) != 3 /*empty*/)
7881 u8Ftw |= RT_BIT(i);
7882 u16FullFtw >>= 2;
7883 }
7884
7885 return u8Ftw;
7886}
7887
7888/** @} */
7889
7890
7891/** @name Memory access.
7892 *
7893 * @{
7894 */
7895
7896
7897/**
7898 * Updates the IEMCPU::cbWritten counter if applicable.
7899 *
7900 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7901 * @param fAccess The access being accounted for.
7902 * @param cbMem The access size.
7903 */
7904DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPUCC pVCpu, uint32_t fAccess, size_t cbMem)
7905{
7906 if ( (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK | IEM_ACCESS_TYPE_WRITE)
7907 || (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA | IEM_ACCESS_TYPE_WRITE) )
7908 pVCpu->iem.s.cbWritten += (uint32_t)cbMem;
7909}
7910
7911
7912/**
7913 * Checks if the given segment can be written to, raise the appropriate
7914 * exception if not.
7915 *
7916 * @returns VBox strict status code.
7917 *
7918 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7919 * @param pHid Pointer to the hidden register.
7920 * @param iSegReg The register number.
7921 * @param pu64BaseAddr Where to return the base address to use for the
7922 * segment. (In 64-bit code it may differ from the
7923 * base in the hidden segment.)
7924 */
7925IEM_STATIC VBOXSTRICTRC
7926iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
7927{
7928 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
7929
7930 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
7931 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
7932 else
7933 {
7934 if (!pHid->Attr.n.u1Present)
7935 {
7936 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
7937 AssertRelease(uSel == 0);
7938 Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
7939 return iemRaiseGeneralProtectionFault0(pVCpu);
7940 }
7941
7942 if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
7943 || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
7944 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
7945 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
7946 *pu64BaseAddr = pHid->u64Base;
7947 }
7948 return VINF_SUCCESS;
7949}
7950
7951
7952/**
7953 * Checks if the given segment can be read from, raise the appropriate
7954 * exception if not.
7955 *
7956 * @returns VBox strict status code.
7957 *
7958 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7959 * @param pHid Pointer to the hidden register.
7960 * @param iSegReg The register number.
7961 * @param pu64BaseAddr Where to return the base address to use for the
7962 * segment. (In 64-bit code it may differ from the
7963 * base in the hidden segment.)
7964 */
7965IEM_STATIC VBOXSTRICTRC
7966iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
7967{
7968 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
7969
7970 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
7971 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
7972 else
7973 {
7974 if (!pHid->Attr.n.u1Present)
7975 {
7976 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
7977 AssertRelease(uSel == 0);
7978 Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
7979 return iemRaiseGeneralProtectionFault0(pVCpu);
7980 }
7981
7982 if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
7983 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
7984 *pu64BaseAddr = pHid->u64Base;
7985 }
7986 return VINF_SUCCESS;
7987}
7988
7989
7990/**
7991 * Applies the segment limit, base and attributes.
7992 *
7993 * This may raise a \#GP or \#SS.
7994 *
7995 * @returns VBox strict status code.
7996 *
7997 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7998 * @param fAccess The kind of access which is being performed.
7999 * @param iSegReg The index of the segment register to apply.
8000 * This is UINT8_MAX if none (for IDT, GDT, LDT,
8001 * TSS, ++).
8002 * @param cbMem The access size.
8003 * @param pGCPtrMem Pointer to the guest memory address to apply
8004 * segmentation to. Input and output parameter.
8005 */
8006IEM_STATIC VBOXSTRICTRC
8007iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem)
8008{
8009 if (iSegReg == UINT8_MAX)
8010 return VINF_SUCCESS;
8011
8012 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
8013 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
8014 switch (pVCpu->iem.s.enmCpuMode)
8015 {
8016 case IEMMODE_16BIT:
8017 case IEMMODE_32BIT:
8018 {
8019 RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
8020 RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
8021
8022 if ( pSel->Attr.n.u1Present
8023 && !pSel->Attr.n.u1Unusable)
8024 {
8025 Assert(pSel->Attr.n.u1DescType);
8026 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
8027 {
8028 if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
8029 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
8030 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
8031
8032 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
8033 {
8034 /** @todo CPL check. */
8035 }
8036
8037 /*
8038 * There are two kinds of data selectors, normal and expand down.
8039 */
8040 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
8041 {
8042 if ( GCPtrFirst32 > pSel->u32Limit
8043 || GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
8044 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
8045 }
8046 else
8047 {
8048 /*
8049 * The upper boundary is defined by the B bit, not the G bit!
8050 */
8051 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
8052 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
8053 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
8054 }
8055 *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
8056 }
8057 else
8058 {
8059
8060 /*
8061 * Code selector and usually be used to read thru, writing is
8062 * only permitted in real and V8086 mode.
8063 */
8064 if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
8065 || ( (fAccess & IEM_ACCESS_TYPE_READ)
8066 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
8067 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )
8068 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
8069
8070 if ( GCPtrFirst32 > pSel->u32Limit
8071 || GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
8072 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
8073
8074 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
8075 {
8076 /** @todo CPL check. */
8077 }
8078
8079 *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
8080 }
8081 }
8082 else
8083 return iemRaiseGeneralProtectionFault0(pVCpu);
8084 return VINF_SUCCESS;
8085 }
8086
8087 case IEMMODE_64BIT:
8088 {
8089 RTGCPTR GCPtrMem = *pGCPtrMem;
8090 if (iSegReg == X86_SREG_GS || iSegReg == X86_SREG_FS)
8091 *pGCPtrMem = GCPtrMem + pSel->u64Base;
8092
8093 Assert(cbMem >= 1);
8094 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
8095 return VINF_SUCCESS;
8096 /** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec.
8097 * 4.12.2 "Data Limit Checks in 64-bit Mode". */
8098 return iemRaiseGeneralProtectionFault0(pVCpu);
8099 }
8100
8101 default:
8102 AssertFailedReturn(VERR_IEM_IPE_7);
8103 }
8104}
8105
8106
8107/**
8108 * Translates a virtual address to a physical physical address and checks if we
8109 * can access the page as specified.
8110 *
8111 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8112 * @param GCPtrMem The virtual address.
8113 * @param fAccess The intended access.
8114 * @param pGCPhysMem Where to return the physical address.
8115 */
8116IEM_STATIC VBOXSTRICTRC
8117iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem)
8118{
8119 /** @todo Need a different PGM interface here. We're currently using
8120 * generic / REM interfaces. this won't cut it for R0. */
8121 /** @todo If/when PGM handles paged real-mode, we can remove the hack in
8122 * iemSvmWorldSwitch/iemVmxWorldSwitch to work around raising a page-fault
8123 * here. */
8124 RTGCPHYS GCPhys;
8125 uint64_t fFlags;
8126 int rc = PGMGstGetPage(pVCpu, GCPtrMem, &fFlags, &GCPhys);
8127 if (RT_FAILURE(rc))
8128 {
8129 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
8130 /** @todo Check unassigned memory in unpaged mode. */
8131 /** @todo Reserved bits in page tables. Requires new PGM interface. */
8132 *pGCPhysMem = NIL_RTGCPHYS;
8133 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, rc);
8134 }
8135
8136 /* If the page is writable and does not have the no-exec bit set, all
8137 access is allowed. Otherwise we'll have to check more carefully... */
8138 if ((fFlags & (X86_PTE_RW | X86_PTE_US | X86_PTE_PAE_NX)) != (X86_PTE_RW | X86_PTE_US))
8139 {
8140 /* Write to read only memory? */
8141 if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
8142 && !(fFlags & X86_PTE_RW)
8143 && ( (pVCpu->iem.s.uCpl == 3
8144 && !(fAccess & IEM_ACCESS_WHAT_SYS))
8145 || (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
8146 {
8147 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
8148 *pGCPhysMem = NIL_RTGCPHYS;
8149 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
8150 }
8151
8152 /* Kernel memory accessed by userland? */
8153 if ( !(fFlags & X86_PTE_US)
8154 && pVCpu->iem.s.uCpl == 3
8155 && !(fAccess & IEM_ACCESS_WHAT_SYS))
8156 {
8157 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
8158 *pGCPhysMem = NIL_RTGCPHYS;
8159 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
8160 }
8161
8162 /* Executing non-executable memory? */
8163 if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
8164 && (fFlags & X86_PTE_PAE_NX)
8165 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) )
8166 {
8167 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
8168 *pGCPhysMem = NIL_RTGCPHYS;
8169 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE),
8170 VERR_ACCESS_DENIED);
8171 }
8172 }
8173
8174 /*
8175 * Set the dirty / access flags.
8176 * ASSUMES this is set when the address is translated rather than on committ...
8177 */
8178 /** @todo testcase: check when A and D bits are actually set by the CPU. */
8179 uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A;
8180 if ((fFlags & fAccessedDirty) != fAccessedDirty)
8181 {
8182 int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
8183 AssertRC(rc2);
8184 }
8185
8186 GCPhys |= GCPtrMem & PAGE_OFFSET_MASK;
8187 *pGCPhysMem = GCPhys;
8188 return VINF_SUCCESS;
8189}
8190
8191
8192
8193/**
8194 * Maps a physical page.
8195 *
8196 * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
8197 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8198 * @param GCPhysMem The physical address.
8199 * @param fAccess The intended access.
8200 * @param ppvMem Where to return the mapping address.
8201 * @param pLock The PGM lock.
8202 */
8203IEM_STATIC int iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem, PPGMPAGEMAPLOCK pLock)
8204{
8205#ifdef IEM_LOG_MEMORY_WRITES
8206 if (fAccess & IEM_ACCESS_TYPE_WRITE)
8207 return VERR_PGM_PHYS_TLB_CATCH_ALL;
8208#endif
8209
8210 /** @todo This API may require some improving later. A private deal with PGM
8211 * regarding locking and unlocking needs to be struct. A couple of TLBs
8212 * living in PGM, but with publicly accessible inlined access methods
8213 * could perhaps be an even better solution. */
8214 int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
8215 GCPhysMem,
8216 RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
8217 pVCpu->iem.s.fBypassHandlers,
8218 ppvMem,
8219 pLock);
8220 /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/
8221 AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc));
8222
8223 return rc;
8224}
8225
8226
8227/**
8228 * Unmap a page previously mapped by iemMemPageMap.
8229 *
8230 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8231 * @param GCPhysMem The physical address.
8232 * @param fAccess The intended access.
8233 * @param pvMem What iemMemPageMap returned.
8234 * @param pLock The PGM lock.
8235 */
8236DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem, PPGMPAGEMAPLOCK pLock)
8237{
8238 NOREF(pVCpu);
8239 NOREF(GCPhysMem);
8240 NOREF(fAccess);
8241 NOREF(pvMem);
8242 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
8243}
8244
8245
8246/**
8247 * Looks up a memory mapping entry.
8248 *
8249 * @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
8250 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8251 * @param pvMem The memory address.
8252 * @param fAccess The access to.
8253 */
8254DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
8255{
8256 Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
8257 fAccess &= IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK;
8258 if ( pVCpu->iem.s.aMemMappings[0].pv == pvMem
8259 && (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
8260 return 0;
8261 if ( pVCpu->iem.s.aMemMappings[1].pv == pvMem
8262 && (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
8263 return 1;
8264 if ( pVCpu->iem.s.aMemMappings[2].pv == pvMem
8265 && (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
8266 return 2;
8267 return VERR_NOT_FOUND;
8268}
8269
8270
8271/**
8272 * Finds a free memmap entry when using iNextMapping doesn't work.
8273 *
8274 * @returns Memory mapping index, 1024 on failure.
8275 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8276 */
8277IEM_STATIC unsigned iemMemMapFindFree(PVMCPUCC pVCpu)
8278{
8279 /*
8280 * The easy case.
8281 */
8282 if (pVCpu->iem.s.cActiveMappings == 0)
8283 {
8284 pVCpu->iem.s.iNextMapping = 1;
8285 return 0;
8286 }
8287
8288 /* There should be enough mappings for all instructions. */
8289 AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);
8290
8291 for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)
8292 if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
8293 return i;
8294
8295 AssertFailedReturn(1024);
8296}
8297
8298
8299/**
8300 * Commits a bounce buffer that needs writing back and unmaps it.
8301 *
8302 * @returns Strict VBox status code.
8303 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8304 * @param iMemMap The index of the buffer to commit.
8305 * @param fPostponeFail Whether we can postpone writer failures to ring-3.
8306 * Always false in ring-3, obviously.
8307 */
8308IEM_STATIC VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail)
8309{
8310 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
8311 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
8312#ifdef IN_RING3
8313 Assert(!fPostponeFail);
8314 RT_NOREF_PV(fPostponeFail);
8315#endif
8316
8317 /*
8318 * Do the writing.
8319 */
8320 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8321 if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)
8322 {
8323 uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
8324 uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
8325 uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
8326 if (!pVCpu->iem.s.fBypassHandlers)
8327 {
8328 /*
8329 * Carefully and efficiently dealing with access handler return
8330 * codes make this a little bloated.
8331 */
8332 VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,
8333 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
8334 pbBuf,
8335 cbFirst,
8336 PGMACCESSORIGIN_IEM);
8337 if (rcStrict == VINF_SUCCESS)
8338 {
8339 if (cbSecond)
8340 {
8341 rcStrict = PGMPhysWrite(pVM,
8342 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
8343 pbBuf + cbFirst,
8344 cbSecond,
8345 PGMACCESSORIGIN_IEM);
8346 if (rcStrict == VINF_SUCCESS)
8347 { /* nothing */ }
8348 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8349 {
8350 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",
8351 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8352 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8353 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8354 }
8355#ifndef IN_RING3
8356 else if (fPostponeFail)
8357 {
8358 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
8359 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8360 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8361 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
8362 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
8363 return iemSetPassUpStatus(pVCpu, rcStrict);
8364 }
8365#endif
8366 else
8367 {
8368 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
8369 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8370 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8371 return rcStrict;
8372 }
8373 }
8374 }
8375 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8376 {
8377 if (!cbSecond)
8378 {
8379 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",
8380 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));
8381 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8382 }
8383 else
8384 {
8385 VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,
8386 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
8387 pbBuf + cbFirst,
8388 cbSecond,
8389 PGMACCESSORIGIN_IEM);
8390 if (rcStrict2 == VINF_SUCCESS)
8391 {
8392 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",
8393 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8394 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
8395 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8396 }
8397 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
8398 {
8399 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",
8400 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8401 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
8402 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
8403 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8404 }
8405#ifndef IN_RING3
8406 else if (fPostponeFail)
8407 {
8408 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
8409 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8410 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8411 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
8412 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
8413 return iemSetPassUpStatus(pVCpu, rcStrict);
8414 }
8415#endif
8416 else
8417 {
8418 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
8419 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8420 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
8421 return rcStrict2;
8422 }
8423 }
8424 }
8425#ifndef IN_RING3
8426 else if (fPostponeFail)
8427 {
8428 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
8429 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8430 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8431 if (!cbSecond)
8432 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST;
8433 else
8434 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND;
8435 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
8436 return iemSetPassUpStatus(pVCpu, rcStrict);
8437 }
8438#endif
8439 else
8440 {
8441 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
8442 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8443 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
8444 return rcStrict;
8445 }
8446 }
8447 else
8448 {
8449 /*
8450 * No access handlers, much simpler.
8451 */
8452 int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);
8453 if (RT_SUCCESS(rc))
8454 {
8455 if (cbSecond)
8456 {
8457 rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);
8458 if (RT_SUCCESS(rc))
8459 { /* likely */ }
8460 else
8461 {
8462 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
8463 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8464 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
8465 return rc;
8466 }
8467 }
8468 }
8469 else
8470 {
8471 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
8472 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,
8473 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
8474 return rc;
8475 }
8476 }
8477 }
8478
8479#if defined(IEM_LOG_MEMORY_WRITES)
8480 Log(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
8481 RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));
8482 if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)
8483 Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
8484 RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),
8485 &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));
8486
8487 size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
8488 g_cbIemWrote = cbWrote;
8489 memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
8490#endif
8491
8492 /*
8493 * Free the mapping entry.
8494 */
8495 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
8496 Assert(pVCpu->iem.s.cActiveMappings != 0);
8497 pVCpu->iem.s.cActiveMappings--;
8498 return VINF_SUCCESS;
8499}
8500
8501
8502/**
8503 * iemMemMap worker that deals with a request crossing pages.
8504 */
8505IEM_STATIC VBOXSTRICTRC
8506iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
8507{
8508 /*
8509 * Do the address translations.
8510 */
8511 RTGCPHYS GCPhysFirst;
8512 VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, fAccess, &GCPhysFirst);
8513 if (rcStrict != VINF_SUCCESS)
8514 return rcStrict;
8515
8516 RTGCPHYS GCPhysSecond;
8517 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)PAGE_OFFSET_MASK,
8518 fAccess, &GCPhysSecond);
8519 if (rcStrict != VINF_SUCCESS)
8520 return rcStrict;
8521 GCPhysSecond &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
8522
8523 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8524
8525 /*
8526 * Read in the current memory content if it's a read, execute or partial
8527 * write access.
8528 */
8529 uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
8530 uint32_t const cbFirstPage = PAGE_SIZE - (GCPhysFirst & PAGE_OFFSET_MASK);
8531 uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
8532
8533 if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
8534 {
8535 if (!pVCpu->iem.s.fBypassHandlers)
8536 {
8537 /*
8538 * Must carefully deal with access handler status codes here,
8539 * makes the code a bit bloated.
8540 */
8541 rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);
8542 if (rcStrict == VINF_SUCCESS)
8543 {
8544 rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
8545 if (rcStrict == VINF_SUCCESS)
8546 { /*likely */ }
8547 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8548 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8549 else
8550 {
8551 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",
8552 GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8553 return rcStrict;
8554 }
8555 }
8556 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8557 {
8558 VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
8559 if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
8560 {
8561 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
8562 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8563 }
8564 else
8565 {
8566 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",
8567 GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));
8568 return rcStrict2;
8569 }
8570 }
8571 else
8572 {
8573 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
8574 GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
8575 return rcStrict;
8576 }
8577 }
8578 else
8579 {
8580 /*
8581 * No informational status codes here, much more straight forward.
8582 */
8583 int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);
8584 if (RT_SUCCESS(rc))
8585 {
8586 Assert(rc == VINF_SUCCESS);
8587 rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
8588 if (RT_SUCCESS(rc))
8589 Assert(rc == VINF_SUCCESS);
8590 else
8591 {
8592 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
8593 return rc;
8594 }
8595 }
8596 else
8597 {
8598 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
8599 return rc;
8600 }
8601 }
8602 }
8603#ifdef VBOX_STRICT
8604 else
8605 memset(pbBuf, 0xcc, cbMem);
8606 if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
8607 memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
8608#endif
8609
8610 /*
8611 * Commit the bounce buffer entry.
8612 */
8613 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
8614 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
8615 pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
8616 pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
8617 pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = false;
8618 pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
8619 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
8620 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8621 pVCpu->iem.s.cActiveMappings++;
8622
8623 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8624 *ppvMem = pbBuf;
8625 return VINF_SUCCESS;
8626}
8627
8628
8629/**
8630 * iemMemMap woker that deals with iemMemPageMap failures.
8631 */
8632IEM_STATIC VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
8633 RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
8634{
8635 /*
8636 * Filter out conditions we can handle and the ones which shouldn't happen.
8637 */
8638 if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
8639 && rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
8640 && rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
8641 {
8642 AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);
8643 return rcMap;
8644 }
8645 pVCpu->iem.s.cPotentialExits++;
8646
8647 /*
8648 * Read in the current memory content if it's a read, execute or partial
8649 * write access.
8650 */
8651 uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
8652 if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
8653 {
8654 if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
8655 memset(pbBuf, 0xff, cbMem);
8656 else
8657 {
8658 int rc;
8659 if (!pVCpu->iem.s.fBypassHandlers)
8660 {
8661 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);
8662 if (rcStrict == VINF_SUCCESS)
8663 { /* nothing */ }
8664 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8665 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8666 else
8667 {
8668 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
8669 GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
8670 return rcStrict;
8671 }
8672 }
8673 else
8674 {
8675 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);
8676 if (RT_SUCCESS(rc))
8677 { /* likely */ }
8678 else
8679 {
8680 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
8681 GCPhysFirst, rc));
8682 return rc;
8683 }
8684 }
8685 }
8686 }
8687#ifdef VBOX_STRICT
8688 else
8689 memset(pbBuf, 0xcc, cbMem);
8690#endif
8691#ifdef VBOX_STRICT
8692 if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
8693 memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
8694#endif
8695
8696 /*
8697 * Commit the bounce buffer entry.
8698 */
8699 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
8700 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
8701 pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
8702 pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = 0;
8703 pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
8704 pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
8705 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
8706 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8707 pVCpu->iem.s.cActiveMappings++;
8708
8709 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8710 *ppvMem = pbBuf;
8711 return VINF_SUCCESS;
8712}
8713
8714
8715
8716/**
8717 * Maps the specified guest memory for the given kind of access.
8718 *
8719 * This may be using bounce buffering of the memory if it's crossing a page
8720 * boundary or if there is an access handler installed for any of it. Because
8721 * of lock prefix guarantees, we're in for some extra clutter when this
8722 * happens.
8723 *
8724 * This may raise a \#GP, \#SS, \#PF or \#AC.
8725 *
8726 * @returns VBox strict status code.
8727 *
8728 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8729 * @param ppvMem Where to return the pointer to the mapped
8730 * memory.
8731 * @param cbMem The number of bytes to map. This is usually 1,
8732 * 2, 4, 6, 8, 12, 16, 32 or 512. When used by
8733 * string operations it can be up to a page.
8734 * @param iSegReg The index of the segment register to use for
8735 * this access. The base and limits are checked.
8736 * Use UINT8_MAX to indicate that no segmentation
8737 * is required (for IDT, GDT and LDT accesses).
8738 * @param GCPtrMem The address of the guest memory.
8739 * @param fAccess How the memory is being accessed. The
8740 * IEM_ACCESS_TYPE_XXX bit is used to figure out
8741 * how to map the memory, while the
8742 * IEM_ACCESS_WHAT_XXX bit is used when raising
8743 * exceptions.
8744 */
8745IEM_STATIC VBOXSTRICTRC
8746iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
8747{
8748 /*
8749 * Check the input and figure out which mapping entry to use.
8750 */
8751 Assert(cbMem <= 64 || cbMem == 512 || cbMem == 256 || cbMem == 108 || cbMem == 104 || cbMem == 102 || cbMem == 94); /* 512 is the max! */
8752 Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
8753 Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
8754
8755 unsigned iMemMap = pVCpu->iem.s.iNextMapping;
8756 if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
8757 || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
8758 {
8759 iMemMap = iemMemMapFindFree(pVCpu);
8760 AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
8761 ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
8762 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
8763 pVCpu->iem.s.aMemMappings[2].fAccess),
8764 VERR_IEM_IPE_9);
8765 }
8766
8767 /*
8768 * Map the memory, checking that we can actually access it. If something
8769 * slightly complicated happens, fall back on bounce buffering.
8770 */
8771 VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
8772 if (rcStrict != VINF_SUCCESS)
8773 return rcStrict;
8774
8775 if ((GCPtrMem & PAGE_OFFSET_MASK) + cbMem > PAGE_SIZE) /* Crossing a page boundary? */
8776 return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
8777
8778 RTGCPHYS GCPhysFirst;
8779 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
8780 if (rcStrict != VINF_SUCCESS)
8781 return rcStrict;
8782
8783 if (fAccess & IEM_ACCESS_TYPE_WRITE)
8784 Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8785 if (fAccess & IEM_ACCESS_TYPE_READ)
8786 Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8787
8788 void *pvMem;
8789 rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8790 if (rcStrict != VINF_SUCCESS)
8791 return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
8792
8793 /*
8794 * Fill in the mapping table entry.
8795 */
8796 pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
8797 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
8798 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8799 pVCpu->iem.s.cActiveMappings++;
8800
8801 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8802 *ppvMem = pvMem;
8803
8804 return VINF_SUCCESS;
8805}
8806
8807
8808/**
8809 * Commits the guest memory if bounce buffered and unmaps it.
8810 *
8811 * @returns Strict VBox status code.
8812 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8813 * @param pvMem The mapping.
8814 * @param fAccess The kind of access.
8815 */
8816IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
8817{
8818 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
8819 AssertReturn(iMemMap >= 0, iMemMap);
8820
8821 /* If it's bounce buffered, we may need to write back the buffer. */
8822 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
8823 {
8824 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
8825 return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
8826 }
8827 /* Otherwise unlock it. */
8828 else
8829 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8830
8831 /* Free the entry. */
8832 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
8833 Assert(pVCpu->iem.s.cActiveMappings != 0);
8834 pVCpu->iem.s.cActiveMappings--;
8835 return VINF_SUCCESS;
8836}
8837
8838#ifdef IEM_WITH_SETJMP
8839
8840/**
8841 * Maps the specified guest memory for the given kind of access, longjmp on
8842 * error.
8843 *
8844 * This may be using bounce buffering of the memory if it's crossing a page
8845 * boundary or if there is an access handler installed for any of it. Because
8846 * of lock prefix guarantees, we're in for some extra clutter when this
8847 * happens.
8848 *
8849 * This may raise a \#GP, \#SS, \#PF or \#AC.
8850 *
8851 * @returns Pointer to the mapped memory.
8852 *
8853 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8854 * @param cbMem The number of bytes to map. This is usually 1,
8855 * 2, 4, 6, 8, 12, 16, 32 or 512. When used by
8856 * string operations it can be up to a page.
8857 * @param iSegReg The index of the segment register to use for
8858 * this access. The base and limits are checked.
8859 * Use UINT8_MAX to indicate that no segmentation
8860 * is required (for IDT, GDT and LDT accesses).
8861 * @param GCPtrMem The address of the guest memory.
8862 * @param fAccess How the memory is being accessed. The
8863 * IEM_ACCESS_TYPE_XXX bit is used to figure out
8864 * how to map the memory, while the
8865 * IEM_ACCESS_WHAT_XXX bit is used when raising
8866 * exceptions.
8867 */
8868IEM_STATIC void *iemMemMapJmp(PVMCPUCC pVCpu, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
8869{
8870 /*
8871 * Check the input and figure out which mapping entry to use.
8872 */
8873 Assert(cbMem <= 64 || cbMem == 512 || cbMem == 108 || cbMem == 104 || cbMem == 94); /* 512 is the max! */
8874 Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
8875 Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
8876
8877 unsigned iMemMap = pVCpu->iem.s.iNextMapping;
8878 if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
8879 || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
8880 {
8881 iMemMap = iemMemMapFindFree(pVCpu);
8882 AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
8883 ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
8884 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
8885 pVCpu->iem.s.aMemMappings[2].fAccess),
8886 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_9));
8887 }
8888
8889 /*
8890 * Map the memory, checking that we can actually access it. If something
8891 * slightly complicated happens, fall back on bounce buffering.
8892 */
8893 VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
8894 if (rcStrict == VINF_SUCCESS) { /*likely*/ }
8895 else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8896
8897 /* Crossing a page boundary? */
8898 if ((GCPtrMem & PAGE_OFFSET_MASK) + cbMem <= PAGE_SIZE)
8899 { /* No (likely). */ }
8900 else
8901 {
8902 void *pvMem;
8903 rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, cbMem, GCPtrMem, fAccess);
8904 if (rcStrict == VINF_SUCCESS)
8905 return pvMem;
8906 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8907 }
8908
8909 RTGCPHYS GCPhysFirst;
8910 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
8911 if (rcStrict == VINF_SUCCESS) { /*likely*/ }
8912 else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8913
8914 if (fAccess & IEM_ACCESS_TYPE_WRITE)
8915 Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8916 if (fAccess & IEM_ACCESS_TYPE_READ)
8917 Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8918
8919 void *pvMem;
8920 rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8921 if (rcStrict == VINF_SUCCESS)
8922 { /* likely */ }
8923 else
8924 {
8925 rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
8926 if (rcStrict == VINF_SUCCESS)
8927 return pvMem;
8928 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8929 }
8930
8931 /*
8932 * Fill in the mapping table entry.
8933 */
8934 pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
8935 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
8936 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8937 pVCpu->iem.s.cActiveMappings++;
8938
8939 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8940 return pvMem;
8941}
8942
8943
8944/**
8945 * Commits the guest memory if bounce buffered and unmaps it, longjmp on error.
8946 *
8947 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8948 * @param pvMem The mapping.
8949 * @param fAccess The kind of access.
8950 */
8951IEM_STATIC void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
8952{
8953 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
8954 AssertStmt(iMemMap >= 0, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), iMemMap));
8955
8956 /* If it's bounce buffered, we may need to write back the buffer. */
8957 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
8958 {
8959 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
8960 {
8961 VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
8962 if (rcStrict == VINF_SUCCESS)
8963 return;
8964 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8965 }
8966 }
8967 /* Otherwise unlock it. */
8968 else
8969 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8970
8971 /* Free the entry. */
8972 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
8973 Assert(pVCpu->iem.s.cActiveMappings != 0);
8974 pVCpu->iem.s.cActiveMappings--;
8975}
8976
8977#endif /* IEM_WITH_SETJMP */
8978
8979#ifndef IN_RING3
8980/**
8981 * Commits the guest memory if bounce buffered and unmaps it, if any bounce
8982 * buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).
8983 *
8984 * Allows the instruction to be completed and retired, while the IEM user will
8985 * return to ring-3 immediately afterwards and do the postponed writes there.
8986 *
8987 * @returns VBox status code (no strict statuses). Caller must check
8988 * VMCPU_FF_IEM before repeating string instructions and similar stuff.
8989 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8990 * @param pvMem The mapping.
8991 * @param fAccess The kind of access.
8992 */
8993IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
8994{
8995 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
8996 AssertReturn(iMemMap >= 0, iMemMap);
8997
8998 /* If it's bounce buffered, we may need to write back the buffer. */
8999 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
9000 {
9001 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
9002 return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /*fPostponeFail*/);
9003 }
9004 /* Otherwise unlock it. */
9005 else
9006 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
9007
9008 /* Free the entry. */
9009 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
9010 Assert(pVCpu->iem.s.cActiveMappings != 0);
9011 pVCpu->iem.s.cActiveMappings--;
9012 return VINF_SUCCESS;
9013}
9014#endif
9015
9016
9017/**
9018 * Rollbacks mappings, releasing page locks and such.
9019 *
9020 * The caller shall only call this after checking cActiveMappings.
9021 *
9022 * @returns Strict VBox status code to pass up.
9023 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9024 */
9025IEM_STATIC void iemMemRollback(PVMCPUCC pVCpu)
9026{
9027 Assert(pVCpu->iem.s.cActiveMappings > 0);
9028
9029 uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
9030 while (iMemMap-- > 0)
9031 {
9032 uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;
9033 if (fAccess != IEM_ACCESS_INVALID)
9034 {
9035 AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));
9036 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
9037 if (!(fAccess & IEM_ACCESS_BOUNCE_BUFFERED))
9038 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
9039 AssertMsg(pVCpu->iem.s.cActiveMappings > 0,
9040 ("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",
9041 iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,
9042 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));
9043 pVCpu->iem.s.cActiveMappings--;
9044 }
9045 }
9046}
9047
9048
9049/**
9050 * Fetches a data byte.
9051 *
9052 * @returns Strict VBox status code.
9053 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9054 * @param pu8Dst Where to return the byte.
9055 * @param iSegReg The index of the segment register to use for
9056 * this access. The base and limits are checked.
9057 * @param GCPtrMem The address of the guest memory.
9058 */
9059IEM_STATIC VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9060{
9061 /* The lazy approach for now... */
9062 uint8_t const *pu8Src;
9063 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Src, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9064 if (rc == VINF_SUCCESS)
9065 {
9066 *pu8Dst = *pu8Src;
9067 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
9068 }
9069 return rc;
9070}
9071
9072
9073#ifdef IEM_WITH_SETJMP
9074/**
9075 * Fetches a data byte, longjmp on error.
9076 *
9077 * @returns The byte.
9078 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9079 * @param iSegReg The index of the segment register to use for
9080 * this access. The base and limits are checked.
9081 * @param GCPtrMem The address of the guest memory.
9082 */
9083DECL_NO_INLINE(IEM_STATIC, uint8_t) iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9084{
9085 /* The lazy approach for now... */
9086 uint8_t const *pu8Src = (uint8_t const *)iemMemMapJmp(pVCpu, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9087 uint8_t const bRet = *pu8Src;
9088 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
9089 return bRet;
9090}
9091#endif /* IEM_WITH_SETJMP */
9092
9093
9094/**
9095 * Fetches a data word.
9096 *
9097 * @returns Strict VBox status code.
9098 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9099 * @param pu16Dst Where to return the word.
9100 * @param iSegReg The index of the segment register to use for
9101 * this access. The base and limits are checked.
9102 * @param GCPtrMem The address of the guest memory.
9103 */
9104IEM_STATIC VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9105{
9106 /* The lazy approach for now... */
9107 uint16_t const *pu16Src;
9108 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9109 if (rc == VINF_SUCCESS)
9110 {
9111 *pu16Dst = *pu16Src;
9112 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
9113 }
9114 return rc;
9115}
9116
9117
9118#ifdef IEM_WITH_SETJMP
9119/**
9120 * Fetches a data word, longjmp on error.
9121 *
9122 * @returns The word
9123 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9124 * @param iSegReg The index of the segment register to use for
9125 * this access. The base and limits are checked.
9126 * @param GCPtrMem The address of the guest memory.
9127 */
9128DECL_NO_INLINE(IEM_STATIC, uint16_t) iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9129{
9130 /* The lazy approach for now... */
9131 uint16_t const *pu16Src = (uint16_t const *)iemMemMapJmp(pVCpu, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9132 uint16_t const u16Ret = *pu16Src;
9133 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
9134 return u16Ret;
9135}
9136#endif
9137
9138
9139/**
9140 * Fetches a data dword.
9141 *
9142 * @returns Strict VBox status code.
9143 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9144 * @param pu32Dst Where to return the dword.
9145 * @param iSegReg The index of the segment register to use for
9146 * this access. The base and limits are checked.
9147 * @param GCPtrMem The address of the guest memory.
9148 */
9149IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9150{
9151 /* The lazy approach for now... */
9152 uint32_t const *pu32Src;
9153 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9154 if (rc == VINF_SUCCESS)
9155 {
9156 *pu32Dst = *pu32Src;
9157 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9158 }
9159 return rc;
9160}
9161
9162
9163/**
9164 * Fetches a data dword and zero extends it to a qword.
9165 *
9166 * @returns Strict VBox status code.
9167 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9168 * @param pu64Dst Where to return the qword.
9169 * @param iSegReg The index of the segment register to use for
9170 * this access. The base and limits are checked.
9171 * @param GCPtrMem The address of the guest memory.
9172 */
9173IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9174{
9175 /* The lazy approach for now... */
9176 uint32_t const *pu32Src;
9177 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9178 if (rc == VINF_SUCCESS)
9179 {
9180 *pu64Dst = *pu32Src;
9181 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9182 }
9183 return rc;
9184}
9185
9186
9187#ifdef IEM_WITH_SETJMP
9188
9189IEM_STATIC RTGCPTR iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
9190{
9191 Assert(cbMem >= 1);
9192 Assert(iSegReg < X86_SREG_COUNT);
9193
9194 /*
9195 * 64-bit mode is simpler.
9196 */
9197 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
9198 {
9199 if (iSegReg >= X86_SREG_FS)
9200 {
9201 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9202 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9203 GCPtrMem += pSel->u64Base;
9204 }
9205
9206 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
9207 return GCPtrMem;
9208 }
9209 /*
9210 * 16-bit and 32-bit segmentation.
9211 */
9212 else
9213 {
9214 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9215 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9216 if ( (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN))
9217 == X86DESCATTR_P /* data, expand up */
9218 || (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ))
9219 == (X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ) /* code, read-only */ )
9220 {
9221 /* expand up */
9222 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9223 if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
9224 && GCPtrLast32 > (uint32_t)GCPtrMem))
9225 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9226 }
9227 else if ( (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN))
9228 == (X86DESCATTR_P | X86_SEL_TYPE_DOWN) /* data, expand down */ )
9229 {
9230 /* expand down */
9231 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9232 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
9233 && GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
9234 && GCPtrLast32 > (uint32_t)GCPtrMem))
9235 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9236 }
9237 else
9238 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
9239 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
9240 }
9241 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
9242}
9243
9244
9245IEM_STATIC RTGCPTR iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
9246{
9247 Assert(cbMem >= 1);
9248 Assert(iSegReg < X86_SREG_COUNT);
9249
9250 /*
9251 * 64-bit mode is simpler.
9252 */
9253 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
9254 {
9255 if (iSegReg >= X86_SREG_FS)
9256 {
9257 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9258 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9259 GCPtrMem += pSel->u64Base;
9260 }
9261
9262 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
9263 return GCPtrMem;
9264 }
9265 /*
9266 * 16-bit and 32-bit segmentation.
9267 */
9268 else
9269 {
9270 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9271 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9272 uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE
9273 | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN);
9274 if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE)) /* data, expand up */
9275 {
9276 /* expand up */
9277 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9278 if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
9279 && GCPtrLast32 > (uint32_t)GCPtrMem))
9280 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9281 }
9282 else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */
9283 {
9284 /* expand down */
9285 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9286 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
9287 && GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
9288 && GCPtrLast32 > (uint32_t)GCPtrMem))
9289 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9290 }
9291 else
9292 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
9293 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
9294 }
9295 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
9296}
9297
9298
9299/**
9300 * Fetches a data dword, longjmp on error, fallback/safe version.
9301 *
9302 * @returns The dword
9303 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9304 * @param iSegReg The index of the segment register to use for
9305 * this access. The base and limits are checked.
9306 * @param GCPtrMem The address of the guest memory.
9307 */
9308IEM_STATIC uint32_t iemMemFetchDataU32SafeJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9309{
9310 uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9311 uint32_t const u32Ret = *pu32Src;
9312 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9313 return u32Ret;
9314}
9315
9316
9317/**
9318 * Fetches a data dword, longjmp on error.
9319 *
9320 * @returns The dword
9321 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9322 * @param iSegReg The index of the segment register to use for
9323 * this access. The base and limits are checked.
9324 * @param GCPtrMem The address of the guest memory.
9325 */
9326DECL_NO_INLINE(IEM_STATIC, uint32_t) iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9327{
9328# ifdef IEM_WITH_DATA_TLB
9329 RTGCPTR GCPtrEff = iemMemApplySegmentToReadJmp(pVCpu, iSegReg, sizeof(uint32_t), GCPtrMem);
9330 if (RT_LIKELY((GCPtrEff & X86_PAGE_OFFSET_MASK) <= X86_PAGE_SIZE - sizeof(uint32_t)))
9331 {
9332 /// @todo more later.
9333 }
9334
9335 return iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
9336# else
9337 /* The lazy approach. */
9338 uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9339 uint32_t const u32Ret = *pu32Src;
9340 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9341 return u32Ret;
9342# endif
9343}
9344#endif
9345
9346
9347#ifdef SOME_UNUSED_FUNCTION
9348/**
9349 * Fetches a data dword and sign extends it to a qword.
9350 *
9351 * @returns Strict VBox status code.
9352 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9353 * @param pu64Dst Where to return the sign extended value.
9354 * @param iSegReg The index of the segment register to use for
9355 * this access. The base and limits are checked.
9356 * @param GCPtrMem The address of the guest memory.
9357 */
9358IEM_STATIC VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9359{
9360 /* The lazy approach for now... */
9361 int32_t const *pi32Src;
9362 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pi32Src, sizeof(*pi32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9363 if (rc == VINF_SUCCESS)
9364 {
9365 *pu64Dst = *pi32Src;
9366 rc = iemMemCommitAndUnmap(pVCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
9367 }
9368#ifdef __GNUC__ /* warning: GCC may be a royal pain */
9369 else
9370 *pu64Dst = 0;
9371#endif
9372 return rc;
9373}
9374#endif
9375
9376
9377/**
9378 * Fetches a data qword.
9379 *
9380 * @returns Strict VBox status code.
9381 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9382 * @param pu64Dst Where to return the qword.
9383 * @param iSegReg The index of the segment register to use for
9384 * this access. The base and limits are checked.
9385 * @param GCPtrMem The address of the guest memory.
9386 */
9387IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9388{
9389 /* The lazy approach for now... */
9390 uint64_t const *pu64Src;
9391 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9392 if (rc == VINF_SUCCESS)
9393 {
9394 *pu64Dst = *pu64Src;
9395 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9396 }
9397 return rc;
9398}
9399
9400
9401#ifdef IEM_WITH_SETJMP
9402/**
9403 * Fetches a data qword, longjmp on error.
9404 *
9405 * @returns The qword.
9406 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9407 * @param iSegReg The index of the segment register to use for
9408 * this access. The base and limits are checked.
9409 * @param GCPtrMem The address of the guest memory.
9410 */
9411DECL_NO_INLINE(IEM_STATIC, uint64_t) iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9412{
9413 /* The lazy approach for now... */
9414 uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9415 uint64_t const u64Ret = *pu64Src;
9416 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9417 return u64Ret;
9418}
9419#endif
9420
9421
9422/**
9423 * Fetches a data qword, aligned at a 16 byte boundrary (for SSE).
9424 *
9425 * @returns Strict VBox status code.
9426 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9427 * @param pu64Dst Where to return the qword.
9428 * @param iSegReg The index of the segment register to use for
9429 * this access. The base and limits are checked.
9430 * @param GCPtrMem The address of the guest memory.
9431 */
9432IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9433{
9434 /* The lazy approach for now... */
9435 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9436 if (RT_UNLIKELY(GCPtrMem & 15))
9437 return iemRaiseGeneralProtectionFault0(pVCpu);
9438
9439 uint64_t const *pu64Src;
9440 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9441 if (rc == VINF_SUCCESS)
9442 {
9443 *pu64Dst = *pu64Src;
9444 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9445 }
9446 return rc;
9447}
9448
9449
9450#ifdef IEM_WITH_SETJMP
9451/**
9452 * Fetches a data qword, longjmp on error.
9453 *
9454 * @returns The qword.
9455 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9456 * @param iSegReg The index of the segment register to use for
9457 * this access. The base and limits are checked.
9458 * @param GCPtrMem The address of the guest memory.
9459 */
9460DECL_NO_INLINE(IEM_STATIC, uint64_t) iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9461{
9462 /* The lazy approach for now... */
9463 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9464 if (RT_LIKELY(!(GCPtrMem & 15)))
9465 {
9466 uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9467 uint64_t const u64Ret = *pu64Src;
9468 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9469 return u64Ret;
9470 }
9471
9472 VBOXSTRICTRC rc = iemRaiseGeneralProtectionFault0(pVCpu);
9473 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rc));
9474}
9475#endif
9476
9477
9478/**
9479 * Fetches a data tword.
9480 *
9481 * @returns Strict VBox status code.
9482 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9483 * @param pr80Dst Where to return the tword.
9484 * @param iSegReg The index of the segment register to use for
9485 * this access. The base and limits are checked.
9486 * @param GCPtrMem The address of the guest memory.
9487 */
9488IEM_STATIC VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9489{
9490 /* The lazy approach for now... */
9491 PCRTFLOAT80U pr80Src;
9492 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pr80Src, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9493 if (rc == VINF_SUCCESS)
9494 {
9495 *pr80Dst = *pr80Src;
9496 rc = iemMemCommitAndUnmap(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
9497 }
9498 return rc;
9499}
9500
9501
9502#ifdef IEM_WITH_SETJMP
9503/**
9504 * Fetches a data tword, longjmp on error.
9505 *
9506 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9507 * @param pr80Dst Where to return the tword.
9508 * @param iSegReg The index of the segment register to use for
9509 * this access. The base and limits are checked.
9510 * @param GCPtrMem The address of the guest memory.
9511 */
9512DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9513{
9514 /* The lazy approach for now... */
9515 PCRTFLOAT80U pr80Src = (PCRTFLOAT80U)iemMemMapJmp(pVCpu, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9516 *pr80Dst = *pr80Src;
9517 iemMemCommitAndUnmapJmp(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
9518}
9519#endif
9520
9521
9522/**
9523 * Fetches a data dqword (double qword), generally SSE related.
9524 *
9525 * @returns Strict VBox status code.
9526 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9527 * @param pu128Dst Where to return the qword.
9528 * @param iSegReg The index of the segment register to use for
9529 * this access. The base and limits are checked.
9530 * @param GCPtrMem The address of the guest memory.
9531 */
9532IEM_STATIC VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9533{
9534 /* The lazy approach for now... */
9535 PCRTUINT128U pu128Src;
9536 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9537 if (rc == VINF_SUCCESS)
9538 {
9539 pu128Dst->au64[0] = pu128Src->au64[0];
9540 pu128Dst->au64[1] = pu128Src->au64[1];
9541 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9542 }
9543 return rc;
9544}
9545
9546
9547#ifdef IEM_WITH_SETJMP
9548/**
9549 * Fetches a data dqword (double qword), generally SSE related.
9550 *
9551 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9552 * @param pu128Dst Where to return the qword.
9553 * @param iSegReg The index of the segment register to use for
9554 * this access. The base and limits are checked.
9555 * @param GCPtrMem The address of the guest memory.
9556 */
9557IEM_STATIC void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9558{
9559 /* The lazy approach for now... */
9560 PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9561 pu128Dst->au64[0] = pu128Src->au64[0];
9562 pu128Dst->au64[1] = pu128Src->au64[1];
9563 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9564}
9565#endif
9566
9567
9568/**
9569 * Fetches a data dqword (double qword) at an aligned address, generally SSE
9570 * related.
9571 *
9572 * Raises \#GP(0) if not aligned.
9573 *
9574 * @returns Strict VBox status code.
9575 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9576 * @param pu128Dst Where to return the qword.
9577 * @param iSegReg The index of the segment register to use for
9578 * this access. The base and limits are checked.
9579 * @param GCPtrMem The address of the guest memory.
9580 */
9581IEM_STATIC VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9582{
9583 /* The lazy approach for now... */
9584 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9585 if ( (GCPtrMem & 15)
9586 && !(pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
9587 return iemRaiseGeneralProtectionFault0(pVCpu);
9588
9589 PCRTUINT128U pu128Src;
9590 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9591 if (rc == VINF_SUCCESS)
9592 {
9593 pu128Dst->au64[0] = pu128Src->au64[0];
9594 pu128Dst->au64[1] = pu128Src->au64[1];
9595 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9596 }
9597 return rc;
9598}
9599
9600
9601#ifdef IEM_WITH_SETJMP
9602/**
9603 * Fetches a data dqword (double qword) at an aligned address, generally SSE
9604 * related, longjmp on error.
9605 *
9606 * Raises \#GP(0) if not aligned.
9607 *
9608 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9609 * @param pu128Dst Where to return the qword.
9610 * @param iSegReg The index of the segment register to use for
9611 * this access. The base and limits are checked.
9612 * @param GCPtrMem The address of the guest memory.
9613 */
9614DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9615{
9616 /* The lazy approach for now... */
9617 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9618 if ( (GCPtrMem & 15) == 0
9619 || (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
9620 {
9621 PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9622 pu128Dst->au64[0] = pu128Src->au64[0];
9623 pu128Dst->au64[1] = pu128Src->au64[1];
9624 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9625 return;
9626 }
9627
9628 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
9629 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
9630}
9631#endif
9632
9633
9634/**
9635 * Fetches a data oword (octo word), generally AVX related.
9636 *
9637 * @returns Strict VBox status code.
9638 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9639 * @param pu256Dst Where to return the qword.
9640 * @param iSegReg The index of the segment register to use for
9641 * this access. The base and limits are checked.
9642 * @param GCPtrMem The address of the guest memory.
9643 */
9644IEM_STATIC VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9645{
9646 /* The lazy approach for now... */
9647 PCRTUINT256U pu256Src;
9648 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9649 if (rc == VINF_SUCCESS)
9650 {
9651 pu256Dst->au64[0] = pu256Src->au64[0];
9652 pu256Dst->au64[1] = pu256Src->au64[1];
9653 pu256Dst->au64[2] = pu256Src->au64[2];
9654 pu256Dst->au64[3] = pu256Src->au64[3];
9655 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9656 }
9657 return rc;
9658}
9659
9660
9661#ifdef IEM_WITH_SETJMP
9662/**
9663 * Fetches a data oword (octo word), generally AVX related.
9664 *
9665 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9666 * @param pu256Dst Where to return the qword.
9667 * @param iSegReg The index of the segment register to use for
9668 * this access. The base and limits are checked.
9669 * @param GCPtrMem The address of the guest memory.
9670 */
9671IEM_STATIC void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9672{
9673 /* The lazy approach for now... */
9674 PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9675 pu256Dst->au64[0] = pu256Src->au64[0];
9676 pu256Dst->au64[1] = pu256Src->au64[1];
9677 pu256Dst->au64[2] = pu256Src->au64[2];
9678 pu256Dst->au64[3] = pu256Src->au64[3];
9679 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9680}
9681#endif
9682
9683
9684/**
9685 * Fetches a data oword (octo word) at an aligned address, generally AVX
9686 * related.
9687 *
9688 * Raises \#GP(0) if not aligned.
9689 *
9690 * @returns Strict VBox status code.
9691 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9692 * @param pu256Dst Where to return the qword.
9693 * @param iSegReg The index of the segment register to use for
9694 * this access. The base and limits are checked.
9695 * @param GCPtrMem The address of the guest memory.
9696 */
9697IEM_STATIC VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9698{
9699 /* The lazy approach for now... */
9700 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on AVX stuff. */
9701 if (GCPtrMem & 31)
9702 return iemRaiseGeneralProtectionFault0(pVCpu);
9703
9704 PCRTUINT256U pu256Src;
9705 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9706 if (rc == VINF_SUCCESS)
9707 {
9708 pu256Dst->au64[0] = pu256Src->au64[0];
9709 pu256Dst->au64[1] = pu256Src->au64[1];
9710 pu256Dst->au64[2] = pu256Src->au64[2];
9711 pu256Dst->au64[3] = pu256Src->au64[3];
9712 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9713 }
9714 return rc;
9715}
9716
9717
9718#ifdef IEM_WITH_SETJMP
9719/**
9720 * Fetches a data oword (octo word) at an aligned address, generally AVX
9721 * related, longjmp on error.
9722 *
9723 * Raises \#GP(0) if not aligned.
9724 *
9725 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9726 * @param pu256Dst Where to return the qword.
9727 * @param iSegReg The index of the segment register to use for
9728 * this access. The base and limits are checked.
9729 * @param GCPtrMem The address of the guest memory.
9730 */
9731DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9732{
9733 /* The lazy approach for now... */
9734 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on AVX stuff. */
9735 if ((GCPtrMem & 31) == 0)
9736 {
9737 PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9738 pu256Dst->au64[0] = pu256Src->au64[0];
9739 pu256Dst->au64[1] = pu256Src->au64[1];
9740 pu256Dst->au64[2] = pu256Src->au64[2];
9741 pu256Dst->au64[3] = pu256Src->au64[3];
9742 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9743 return;
9744 }
9745
9746 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
9747 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
9748}
9749#endif
9750
9751
9752
9753/**
9754 * Fetches a descriptor register (lgdt, lidt).
9755 *
9756 * @returns Strict VBox status code.
9757 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9758 * @param pcbLimit Where to return the limit.
9759 * @param pGCPtrBase Where to return the base.
9760 * @param iSegReg The index of the segment register to use for
9761 * this access. The base and limits are checked.
9762 * @param GCPtrMem The address of the guest memory.
9763 * @param enmOpSize The effective operand size.
9764 */
9765IEM_STATIC VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
9766 RTGCPTR GCPtrMem, IEMMODE enmOpSize)
9767{
9768 /*
9769 * Just like SIDT and SGDT, the LIDT and LGDT instructions are a
9770 * little special:
9771 * - The two reads are done separately.
9772 * - Operand size override works in 16-bit and 32-bit code, but 64-bit.
9773 * - We suspect the 386 to actually commit the limit before the base in
9774 * some cases (search for 386 in bs3CpuBasic2_lidt_lgdt_One). We
9775 * don't try emulate this eccentric behavior, because it's not well
9776 * enough understood and rather hard to trigger.
9777 * - The 486 seems to do a dword limit read when the operand size is 32-bit.
9778 */
9779 VBOXSTRICTRC rcStrict;
9780 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
9781 {
9782 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
9783 if (rcStrict == VINF_SUCCESS)
9784 rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2);
9785 }
9786 else
9787 {
9788 uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */
9789 if (enmOpSize == IEMMODE_32BIT)
9790 {
9791 if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486)
9792 {
9793 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
9794 if (rcStrict == VINF_SUCCESS)
9795 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
9796 }
9797 else
9798 {
9799 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem);
9800 if (rcStrict == VINF_SUCCESS)
9801 {
9802 *pcbLimit = (uint16_t)uTmp;
9803 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
9804 }
9805 }
9806 if (rcStrict == VINF_SUCCESS)
9807 *pGCPtrBase = uTmp;
9808 }
9809 else
9810 {
9811 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
9812 if (rcStrict == VINF_SUCCESS)
9813 {
9814 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
9815 if (rcStrict == VINF_SUCCESS)
9816 *pGCPtrBase = uTmp & UINT32_C(0x00ffffff);
9817 }
9818 }
9819 }
9820 return rcStrict;
9821}
9822
9823
9824
9825/**
9826 * Stores a data byte.
9827 *
9828 * @returns Strict VBox status code.
9829 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9830 * @param iSegReg The index of the segment register to use for
9831 * this access. The base and limits are checked.
9832 * @param GCPtrMem The address of the guest memory.
9833 * @param u8Value The value to store.
9834 */
9835IEM_STATIC VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
9836{
9837 /* The lazy approach for now... */
9838 uint8_t *pu8Dst;
9839 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Dst, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9840 if (rc == VINF_SUCCESS)
9841 {
9842 *pu8Dst = u8Value;
9843 rc = iemMemCommitAndUnmap(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
9844 }
9845 return rc;
9846}
9847
9848
9849#ifdef IEM_WITH_SETJMP
9850/**
9851 * Stores a data byte, longjmp on error.
9852 *
9853 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9854 * @param iSegReg The index of the segment register to use for
9855 * this access. The base and limits are checked.
9856 * @param GCPtrMem The address of the guest memory.
9857 * @param u8Value The value to store.
9858 */
9859IEM_STATIC void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
9860{
9861 /* The lazy approach for now... */
9862 uint8_t *pu8Dst = (uint8_t *)iemMemMapJmp(pVCpu, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9863 *pu8Dst = u8Value;
9864 iemMemCommitAndUnmapJmp(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
9865}
9866#endif
9867
9868
9869/**
9870 * Stores a data word.
9871 *
9872 * @returns Strict VBox status code.
9873 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9874 * @param iSegReg The index of the segment register to use for
9875 * this access. The base and limits are checked.
9876 * @param GCPtrMem The address of the guest memory.
9877 * @param u16Value The value to store.
9878 */
9879IEM_STATIC VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
9880{
9881 /* The lazy approach for now... */
9882 uint16_t *pu16Dst;
9883 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9884 if (rc == VINF_SUCCESS)
9885 {
9886 *pu16Dst = u16Value;
9887 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
9888 }
9889 return rc;
9890}
9891
9892
9893#ifdef IEM_WITH_SETJMP
9894/**
9895 * Stores a data word, longjmp on error.
9896 *
9897 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9898 * @param iSegReg The index of the segment register to use for
9899 * this access. The base and limits are checked.
9900 * @param GCPtrMem The address of the guest memory.
9901 * @param u16Value The value to store.
9902 */
9903IEM_STATIC void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
9904{
9905 /* The lazy approach for now... */
9906 uint16_t *pu16Dst = (uint16_t *)iemMemMapJmp(pVCpu, sizeof(*pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9907 *pu16Dst = u16Value;
9908 iemMemCommitAndUnmapJmp(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
9909}
9910#endif
9911
9912
9913/**
9914 * Stores a data dword.
9915 *
9916 * @returns Strict VBox status code.
9917 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9918 * @param iSegReg The index of the segment register to use for
9919 * this access. The base and limits are checked.
9920 * @param GCPtrMem The address of the guest memory.
9921 * @param u32Value The value to store.
9922 */
9923IEM_STATIC VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
9924{
9925 /* The lazy approach for now... */
9926 uint32_t *pu32Dst;
9927 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9928 if (rc == VINF_SUCCESS)
9929 {
9930 *pu32Dst = u32Value;
9931 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
9932 }
9933 return rc;
9934}
9935
9936
9937#ifdef IEM_WITH_SETJMP
9938/**
9939 * Stores a data dword.
9940 *
9941 * @returns Strict VBox status code.
9942 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9943 * @param iSegReg The index of the segment register to use for
9944 * this access. The base and limits are checked.
9945 * @param GCPtrMem The address of the guest memory.
9946 * @param u32Value The value to store.
9947 */
9948IEM_STATIC void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
9949{
9950 /* The lazy approach for now... */
9951 uint32_t *pu32Dst = (uint32_t *)iemMemMapJmp(pVCpu, sizeof(*pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9952 *pu32Dst = u32Value;
9953 iemMemCommitAndUnmapJmp(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
9954}
9955#endif
9956
9957
9958/**
9959 * Stores a data qword.
9960 *
9961 * @returns Strict VBox status code.
9962 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9963 * @param iSegReg The index of the segment register to use for
9964 * this access. The base and limits are checked.
9965 * @param GCPtrMem The address of the guest memory.
9966 * @param u64Value The value to store.
9967 */
9968IEM_STATIC VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
9969{
9970 /* The lazy approach for now... */
9971 uint64_t *pu64Dst;
9972 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9973 if (rc == VINF_SUCCESS)
9974 {
9975 *pu64Dst = u64Value;
9976 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
9977 }
9978 return rc;
9979}
9980
9981
9982#ifdef IEM_WITH_SETJMP
9983/**
9984 * Stores a data qword, longjmp on error.
9985 *
9986 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9987 * @param iSegReg The index of the segment register to use for
9988 * this access. The base and limits are checked.
9989 * @param GCPtrMem The address of the guest memory.
9990 * @param u64Value The value to store.
9991 */
9992IEM_STATIC void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
9993{
9994 /* The lazy approach for now... */
9995 uint64_t *pu64Dst = (uint64_t *)iemMemMapJmp(pVCpu, sizeof(*pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9996 *pu64Dst = u64Value;
9997 iemMemCommitAndUnmapJmp(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
9998}
9999#endif
10000
10001
10002/**
10003 * Stores a data dqword.
10004 *
10005 * @returns Strict VBox status code.
10006 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10007 * @param iSegReg The index of the segment register to use for
10008 * this access. The base and limits are checked.
10009 * @param GCPtrMem The address of the guest memory.
10010 * @param u128Value The value to store.
10011 */
10012IEM_STATIC VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10013{
10014 /* The lazy approach for now... */
10015 PRTUINT128U pu128Dst;
10016 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10017 if (rc == VINF_SUCCESS)
10018 {
10019 pu128Dst->au64[0] = u128Value.au64[0];
10020 pu128Dst->au64[1] = u128Value.au64[1];
10021 rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10022 }
10023 return rc;
10024}
10025
10026
10027#ifdef IEM_WITH_SETJMP
10028/**
10029 * Stores a data dqword, longjmp on error.
10030 *
10031 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10032 * @param iSegReg The index of the segment register to use for
10033 * this access. The base and limits are checked.
10034 * @param GCPtrMem The address of the guest memory.
10035 * @param u128Value The value to store.
10036 */
10037IEM_STATIC void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10038{
10039 /* The lazy approach for now... */
10040 PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10041 pu128Dst->au64[0] = u128Value.au64[0];
10042 pu128Dst->au64[1] = u128Value.au64[1];
10043 iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10044}
10045#endif
10046
10047
10048/**
10049 * Stores a data dqword, SSE aligned.
10050 *
10051 * @returns Strict VBox status code.
10052 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10053 * @param iSegReg The index of the segment register to use for
10054 * this access. The base and limits are checked.
10055 * @param GCPtrMem The address of the guest memory.
10056 * @param u128Value The value to store.
10057 */
10058IEM_STATIC VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10059{
10060 /* The lazy approach for now... */
10061 if ( (GCPtrMem & 15)
10062 && !(pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
10063 return iemRaiseGeneralProtectionFault0(pVCpu);
10064
10065 PRTUINT128U pu128Dst;
10066 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10067 if (rc == VINF_SUCCESS)
10068 {
10069 pu128Dst->au64[0] = u128Value.au64[0];
10070 pu128Dst->au64[1] = u128Value.au64[1];
10071 rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10072 }
10073 return rc;
10074}
10075
10076
10077#ifdef IEM_WITH_SETJMP
10078/**
10079 * Stores a data dqword, SSE aligned.
10080 *
10081 * @returns Strict VBox status code.
10082 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10083 * @param iSegReg The index of the segment register to use for
10084 * this access. The base and limits are checked.
10085 * @param GCPtrMem The address of the guest memory.
10086 * @param u128Value The value to store.
10087 */
10088DECL_NO_INLINE(IEM_STATIC, void)
10089iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10090{
10091 /* The lazy approach for now... */
10092 if ( (GCPtrMem & 15) == 0
10093 || (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
10094 {
10095 PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10096 pu128Dst->au64[0] = u128Value.au64[0];
10097 pu128Dst->au64[1] = u128Value.au64[1];
10098 iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10099 return;
10100 }
10101
10102 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
10103 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
10104}
10105#endif
10106
10107
10108/**
10109 * Stores a data dqword.
10110 *
10111 * @returns Strict VBox status code.
10112 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10113 * @param iSegReg The index of the segment register to use for
10114 * this access. The base and limits are checked.
10115 * @param GCPtrMem The address of the guest memory.
10116 * @param pu256Value Pointer to the value to store.
10117 */
10118IEM_STATIC VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10119{
10120 /* The lazy approach for now... */
10121 PRTUINT256U pu256Dst;
10122 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10123 if (rc == VINF_SUCCESS)
10124 {
10125 pu256Dst->au64[0] = pu256Value->au64[0];
10126 pu256Dst->au64[1] = pu256Value->au64[1];
10127 pu256Dst->au64[2] = pu256Value->au64[2];
10128 pu256Dst->au64[3] = pu256Value->au64[3];
10129 rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10130 }
10131 return rc;
10132}
10133
10134
10135#ifdef IEM_WITH_SETJMP
10136/**
10137 * Stores a data dqword, longjmp on error.
10138 *
10139 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10140 * @param iSegReg The index of the segment register to use for
10141 * this access. The base and limits are checked.
10142 * @param GCPtrMem The address of the guest memory.
10143 * @param pu256Value Pointer to the value to store.
10144 */
10145IEM_STATIC void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10146{
10147 /* The lazy approach for now... */
10148 PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10149 pu256Dst->au64[0] = pu256Value->au64[0];
10150 pu256Dst->au64[1] = pu256Value->au64[1];
10151 pu256Dst->au64[2] = pu256Value->au64[2];
10152 pu256Dst->au64[3] = pu256Value->au64[3];
10153 iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10154}
10155#endif
10156
10157
10158/**
10159 * Stores a data dqword, AVX aligned.
10160 *
10161 * @returns Strict VBox status code.
10162 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10163 * @param iSegReg The index of the segment register to use for
10164 * this access. The base and limits are checked.
10165 * @param GCPtrMem The address of the guest memory.
10166 * @param pu256Value Pointer to the value to store.
10167 */
10168IEM_STATIC VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10169{
10170 /* The lazy approach for now... */
10171 if (GCPtrMem & 31)
10172 return iemRaiseGeneralProtectionFault0(pVCpu);
10173
10174 PRTUINT256U pu256Dst;
10175 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10176 if (rc == VINF_SUCCESS)
10177 {
10178 pu256Dst->au64[0] = pu256Value->au64[0];
10179 pu256Dst->au64[1] = pu256Value->au64[1];
10180 pu256Dst->au64[2] = pu256Value->au64[2];
10181 pu256Dst->au64[3] = pu256Value->au64[3];
10182 rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10183 }
10184 return rc;
10185}
10186
10187
10188#ifdef IEM_WITH_SETJMP
10189/**
10190 * Stores a data dqword, AVX aligned.
10191 *
10192 * @returns Strict VBox status code.
10193 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10194 * @param iSegReg The index of the segment register to use for
10195 * this access. The base and limits are checked.
10196 * @param GCPtrMem The address of the guest memory.
10197 * @param pu256Value Pointer to the value to store.
10198 */
10199DECL_NO_INLINE(IEM_STATIC, void)
10200iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10201{
10202 /* The lazy approach for now... */
10203 if ((GCPtrMem & 31) == 0)
10204 {
10205 PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10206 pu256Dst->au64[0] = pu256Value->au64[0];
10207 pu256Dst->au64[1] = pu256Value->au64[1];
10208 pu256Dst->au64[2] = pu256Value->au64[2];
10209 pu256Dst->au64[3] = pu256Value->au64[3];
10210 iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10211 return;
10212 }
10213
10214 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
10215 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
10216}
10217#endif
10218
10219
10220/**
10221 * Stores a descriptor register (sgdt, sidt).
10222 *
10223 * @returns Strict VBox status code.
10224 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10225 * @param cbLimit The limit.
10226 * @param GCPtrBase The base address.
10227 * @param iSegReg The index of the segment register to use for
10228 * this access. The base and limits are checked.
10229 * @param GCPtrMem The address of the guest memory.
10230 */
10231IEM_STATIC VBOXSTRICTRC
10232iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem)
10233{
10234 /*
10235 * The SIDT and SGDT instructions actually stores the data using two
10236 * independent writes. The instructions does not respond to opsize prefixes.
10237 */
10238 VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit);
10239 if (rcStrict == VINF_SUCCESS)
10240 {
10241 if (pVCpu->iem.s.enmCpuMode == IEMMODE_16BIT)
10242 rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2,
10243 IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286
10244 ? (uint32_t)GCPtrBase | UINT32_C(0xff000000) : (uint32_t)GCPtrBase);
10245 else if (pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT)
10246 rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase);
10247 else
10248 rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase);
10249 }
10250 return rcStrict;
10251}
10252
10253
10254/**
10255 * Pushes a word onto the stack.
10256 *
10257 * @returns Strict VBox status code.
10258 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10259 * @param u16Value The value to push.
10260 */
10261IEM_STATIC VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value)
10262{
10263 /* Increment the stack pointer. */
10264 uint64_t uNewRsp;
10265 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 2, &uNewRsp);
10266
10267 /* Write the word the lazy way. */
10268 uint16_t *pu16Dst;
10269 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10270 if (rc == VINF_SUCCESS)
10271 {
10272 *pu16Dst = u16Value;
10273 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
10274 }
10275
10276 /* Commit the new RSP value unless we an access handler made trouble. */
10277 if (rc == VINF_SUCCESS)
10278 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10279
10280 return rc;
10281}
10282
10283
10284/**
10285 * Pushes a dword onto the stack.
10286 *
10287 * @returns Strict VBox status code.
10288 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10289 * @param u32Value The value to push.
10290 */
10291IEM_STATIC VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value)
10292{
10293 /* Increment the stack pointer. */
10294 uint64_t uNewRsp;
10295 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
10296
10297 /* Write the dword the lazy way. */
10298 uint32_t *pu32Dst;
10299 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10300 if (rc == VINF_SUCCESS)
10301 {
10302 *pu32Dst = u32Value;
10303 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
10304 }
10305
10306 /* Commit the new RSP value unless we an access handler made trouble. */
10307 if (rc == VINF_SUCCESS)
10308 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10309
10310 return rc;
10311}
10312
10313
10314/**
10315 * Pushes a dword segment register value onto the stack.
10316 *
10317 * @returns Strict VBox status code.
10318 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10319 * @param u32Value The value to push.
10320 */
10321IEM_STATIC VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value)
10322{
10323 /* Increment the stack pointer. */
10324 uint64_t uNewRsp;
10325 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
10326
10327 /* The intel docs talks about zero extending the selector register
10328 value. My actual intel CPU here might be zero extending the value
10329 but it still only writes the lower word... */
10330 /** @todo Test this on new HW and on AMD and in 64-bit mode. Also test what
10331 * happens when crossing an electric page boundrary, is the high word checked
10332 * for write accessibility or not? Probably it is. What about segment limits?
10333 * It appears this behavior is also shared with trap error codes.
10334 *
10335 * Docs indicate the behavior changed maybe in Pentium or Pentium Pro. Check
10336 * ancient hardware when it actually did change. */
10337 uint16_t *pu16Dst;
10338 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_RW);
10339 if (rc == VINF_SUCCESS)
10340 {
10341 *pu16Dst = (uint16_t)u32Value;
10342 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_RW);
10343 }
10344
10345 /* Commit the new RSP value unless we an access handler made trouble. */
10346 if (rc == VINF_SUCCESS)
10347 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10348
10349 return rc;
10350}
10351
10352
10353/**
10354 * Pushes a qword onto the stack.
10355 *
10356 * @returns Strict VBox status code.
10357 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10358 * @param u64Value The value to push.
10359 */
10360IEM_STATIC VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value)
10361{
10362 /* Increment the stack pointer. */
10363 uint64_t uNewRsp;
10364 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 8, &uNewRsp);
10365
10366 /* Write the word the lazy way. */
10367 uint64_t *pu64Dst;
10368 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10369 if (rc == VINF_SUCCESS)
10370 {
10371 *pu64Dst = u64Value;
10372 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
10373 }
10374
10375 /* Commit the new RSP value unless we an access handler made trouble. */
10376 if (rc == VINF_SUCCESS)
10377 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10378
10379 return rc;
10380}
10381
10382
10383/**
10384 * Pops a word from the stack.
10385 *
10386 * @returns Strict VBox status code.
10387 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10388 * @param pu16Value Where to store the popped value.
10389 */
10390IEM_STATIC VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value)
10391{
10392 /* Increment the stack pointer. */
10393 uint64_t uNewRsp;
10394 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 2, &uNewRsp);
10395
10396 /* Write the word the lazy way. */
10397 uint16_t const *pu16Src;
10398 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10399 if (rc == VINF_SUCCESS)
10400 {
10401 *pu16Value = *pu16Src;
10402 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
10403
10404 /* Commit the new RSP value. */
10405 if (rc == VINF_SUCCESS)
10406 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10407 }
10408
10409 return rc;
10410}
10411
10412
10413/**
10414 * Pops a dword from the stack.
10415 *
10416 * @returns Strict VBox status code.
10417 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10418 * @param pu32Value Where to store the popped value.
10419 */
10420IEM_STATIC VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value)
10421{
10422 /* Increment the stack pointer. */
10423 uint64_t uNewRsp;
10424 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 4, &uNewRsp);
10425
10426 /* Write the word the lazy way. */
10427 uint32_t const *pu32Src;
10428 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10429 if (rc == VINF_SUCCESS)
10430 {
10431 *pu32Value = *pu32Src;
10432 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
10433
10434 /* Commit the new RSP value. */
10435 if (rc == VINF_SUCCESS)
10436 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10437 }
10438
10439 return rc;
10440}
10441
10442
10443/**
10444 * Pops a qword from the stack.
10445 *
10446 * @returns Strict VBox status code.
10447 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10448 * @param pu64Value Where to store the popped value.
10449 */
10450IEM_STATIC VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value)
10451{
10452 /* Increment the stack pointer. */
10453 uint64_t uNewRsp;
10454 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 8, &uNewRsp);
10455
10456 /* Write the word the lazy way. */
10457 uint64_t const *pu64Src;
10458 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10459 if (rc == VINF_SUCCESS)
10460 {
10461 *pu64Value = *pu64Src;
10462 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
10463
10464 /* Commit the new RSP value. */
10465 if (rc == VINF_SUCCESS)
10466 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10467 }
10468
10469 return rc;
10470}
10471
10472
10473/**
10474 * Pushes a word onto the stack, using a temporary stack pointer.
10475 *
10476 * @returns Strict VBox status code.
10477 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10478 * @param u16Value The value to push.
10479 * @param pTmpRsp Pointer to the temporary stack pointer.
10480 */
10481IEM_STATIC VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp)
10482{
10483 /* Increment the stack pointer. */
10484 RTUINT64U NewRsp = *pTmpRsp;
10485 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 2);
10486
10487 /* Write the word the lazy way. */
10488 uint16_t *pu16Dst;
10489 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10490 if (rc == VINF_SUCCESS)
10491 {
10492 *pu16Dst = u16Value;
10493 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
10494 }
10495
10496 /* Commit the new RSP value unless we an access handler made trouble. */
10497 if (rc == VINF_SUCCESS)
10498 *pTmpRsp = NewRsp;
10499
10500 return rc;
10501}
10502
10503
10504/**
10505 * Pushes a dword onto the stack, using a temporary stack pointer.
10506 *
10507 * @returns Strict VBox status code.
10508 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10509 * @param u32Value The value to push.
10510 * @param pTmpRsp Pointer to the temporary stack pointer.
10511 */
10512IEM_STATIC VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp)
10513{
10514 /* Increment the stack pointer. */
10515 RTUINT64U NewRsp = *pTmpRsp;
10516 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 4);
10517
10518 /* Write the word the lazy way. */
10519 uint32_t *pu32Dst;
10520 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10521 if (rc == VINF_SUCCESS)
10522 {
10523 *pu32Dst = u32Value;
10524 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
10525 }
10526
10527 /* Commit the new RSP value unless we an access handler made trouble. */
10528 if (rc == VINF_SUCCESS)
10529 *pTmpRsp = NewRsp;
10530
10531 return rc;
10532}
10533
10534
10535/**
10536 * Pushes a dword onto the stack, using a temporary stack pointer.
10537 *
10538 * @returns Strict VBox status code.
10539 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10540 * @param u64Value The value to push.
10541 * @param pTmpRsp Pointer to the temporary stack pointer.
10542 */
10543IEM_STATIC VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp)
10544{
10545 /* Increment the stack pointer. */
10546 RTUINT64U NewRsp = *pTmpRsp;
10547 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 8);
10548
10549 /* Write the word the lazy way. */
10550 uint64_t *pu64Dst;
10551 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10552 if (rc == VINF_SUCCESS)
10553 {
10554 *pu64Dst = u64Value;
10555 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
10556 }
10557
10558 /* Commit the new RSP value unless we an access handler made trouble. */
10559 if (rc == VINF_SUCCESS)
10560 *pTmpRsp = NewRsp;
10561
10562 return rc;
10563}
10564
10565
10566/**
10567 * Pops a word from the stack, using a temporary stack pointer.
10568 *
10569 * @returns Strict VBox status code.
10570 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10571 * @param pu16Value Where to store the popped value.
10572 * @param pTmpRsp Pointer to the temporary stack pointer.
10573 */
10574IEM_STATIC VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp)
10575{
10576 /* Increment the stack pointer. */
10577 RTUINT64U NewRsp = *pTmpRsp;
10578 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 2);
10579
10580 /* Write the word the lazy way. */
10581 uint16_t const *pu16Src;
10582 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10583 if (rc == VINF_SUCCESS)
10584 {
10585 *pu16Value = *pu16Src;
10586 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
10587
10588 /* Commit the new RSP value. */
10589 if (rc == VINF_SUCCESS)
10590 *pTmpRsp = NewRsp;
10591 }
10592
10593 return rc;
10594}
10595
10596
10597/**
10598 * Pops a dword from the stack, using a temporary stack pointer.
10599 *
10600 * @returns Strict VBox status code.
10601 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10602 * @param pu32Value Where to store the popped value.
10603 * @param pTmpRsp Pointer to the temporary stack pointer.
10604 */
10605IEM_STATIC VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp)
10606{
10607 /* Increment the stack pointer. */
10608 RTUINT64U NewRsp = *pTmpRsp;
10609 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 4);
10610
10611 /* Write the word the lazy way. */
10612 uint32_t const *pu32Src;
10613 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10614 if (rc == VINF_SUCCESS)
10615 {
10616 *pu32Value = *pu32Src;
10617 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
10618
10619 /* Commit the new RSP value. */
10620 if (rc == VINF_SUCCESS)
10621 *pTmpRsp = NewRsp;
10622 }
10623
10624 return rc;
10625}
10626
10627
10628/**
10629 * Pops a qword from the stack, using a temporary stack pointer.
10630 *
10631 * @returns Strict VBox status code.
10632 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10633 * @param pu64Value Where to store the popped value.
10634 * @param pTmpRsp Pointer to the temporary stack pointer.
10635 */
10636IEM_STATIC VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp)
10637{
10638 /* Increment the stack pointer. */
10639 RTUINT64U NewRsp = *pTmpRsp;
10640 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
10641
10642 /* Write the word the lazy way. */
10643 uint64_t const *pu64Src;
10644 VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10645 if (rcStrict == VINF_SUCCESS)
10646 {
10647 *pu64Value = *pu64Src;
10648 rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
10649
10650 /* Commit the new RSP value. */
10651 if (rcStrict == VINF_SUCCESS)
10652 *pTmpRsp = NewRsp;
10653 }
10654
10655 return rcStrict;
10656}
10657
10658
10659/**
10660 * Begin a special stack push (used by interrupt, exceptions and such).
10661 *
10662 * This will raise \#SS or \#PF if appropriate.
10663 *
10664 * @returns Strict VBox status code.
10665 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10666 * @param cbMem The number of bytes to push onto the stack.
10667 * @param ppvMem Where to return the pointer to the stack memory.
10668 * As with the other memory functions this could be
10669 * direct access or bounce buffered access, so
10670 * don't commit register until the commit call
10671 * succeeds.
10672 * @param puNewRsp Where to return the new RSP value. This must be
10673 * passed unchanged to
10674 * iemMemStackPushCommitSpecial().
10675 */
10676IEM_STATIC VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void **ppvMem, uint64_t *puNewRsp)
10677{
10678 Assert(cbMem < UINT8_MAX);
10679 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp);
10680 return iemMemMap(pVCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10681}
10682
10683
10684/**
10685 * Commits a special stack push (started by iemMemStackPushBeginSpecial).
10686 *
10687 * This will update the rSP.
10688 *
10689 * @returns Strict VBox status code.
10690 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10691 * @param pvMem The pointer returned by
10692 * iemMemStackPushBeginSpecial().
10693 * @param uNewRsp The new RSP value returned by
10694 * iemMemStackPushBeginSpecial().
10695 */
10696IEM_STATIC VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp)
10697{
10698 VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem, IEM_ACCESS_STACK_W);
10699 if (rcStrict == VINF_SUCCESS)
10700 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10701 return rcStrict;
10702}
10703
10704
10705/**
10706 * Begin a special stack pop (used by iret, retf and such).
10707 *
10708 * This will raise \#SS or \#PF if appropriate.
10709 *
10710 * @returns Strict VBox status code.
10711 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10712 * @param cbMem The number of bytes to pop from the stack.
10713 * @param ppvMem Where to return the pointer to the stack memory.
10714 * @param puNewRsp Where to return the new RSP value. This must be
10715 * assigned to CPUMCTX::rsp manually some time
10716 * after iemMemStackPopDoneSpecial() has been
10717 * called.
10718 */
10719IEM_STATIC VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp)
10720{
10721 Assert(cbMem < UINT8_MAX);
10722 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp);
10723 return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10724}
10725
10726
10727/**
10728 * Continue a special stack pop (used by iret and retf).
10729 *
10730 * This will raise \#SS or \#PF if appropriate.
10731 *
10732 * @returns Strict VBox status code.
10733 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10734 * @param cbMem The number of bytes to pop from the stack.
10735 * @param ppvMem Where to return the pointer to the stack memory.
10736 * @param puNewRsp Where to return the new RSP value. This must be
10737 * assigned to CPUMCTX::rsp manually some time
10738 * after iemMemStackPopDoneSpecial() has been
10739 * called.
10740 */
10741IEM_STATIC VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp)
10742{
10743 Assert(cbMem < UINT8_MAX);
10744 RTUINT64U NewRsp;
10745 NewRsp.u = *puNewRsp;
10746 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
10747 *puNewRsp = NewRsp.u;
10748 return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10749}
10750
10751
10752/**
10753 * Done with a special stack pop (started by iemMemStackPopBeginSpecial or
10754 * iemMemStackPopContinueSpecial).
10755 *
10756 * The caller will manually commit the rSP.
10757 *
10758 * @returns Strict VBox status code.
10759 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10760 * @param pvMem The pointer returned by
10761 * iemMemStackPopBeginSpecial() or
10762 * iemMemStackPopContinueSpecial().
10763 */
10764IEM_STATIC VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem)
10765{
10766 return iemMemCommitAndUnmap(pVCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
10767}
10768
10769
10770/**
10771 * Fetches a system table byte.
10772 *
10773 * @returns Strict VBox status code.
10774 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10775 * @param pbDst Where to return the byte.
10776 * @param iSegReg The index of the segment register to use for
10777 * this access. The base and limits are checked.
10778 * @param GCPtrMem The address of the guest memory.
10779 */
10780IEM_STATIC VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10781{
10782 /* The lazy approach for now... */
10783 uint8_t const *pbSrc;
10784 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pbSrc, sizeof(*pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10785 if (rc == VINF_SUCCESS)
10786 {
10787 *pbDst = *pbSrc;
10788 rc = iemMemCommitAndUnmap(pVCpu, (void *)pbSrc, IEM_ACCESS_SYS_R);
10789 }
10790 return rc;
10791}
10792
10793
10794/**
10795 * Fetches a system table word.
10796 *
10797 * @returns Strict VBox status code.
10798 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10799 * @param pu16Dst Where to return the word.
10800 * @param iSegReg The index of the segment register to use for
10801 * this access. The base and limits are checked.
10802 * @param GCPtrMem The address of the guest memory.
10803 */
10804IEM_STATIC VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10805{
10806 /* The lazy approach for now... */
10807 uint16_t const *pu16Src;
10808 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10809 if (rc == VINF_SUCCESS)
10810 {
10811 *pu16Dst = *pu16Src;
10812 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_SYS_R);
10813 }
10814 return rc;
10815}
10816
10817
10818/**
10819 * Fetches a system table dword.
10820 *
10821 * @returns Strict VBox status code.
10822 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10823 * @param pu32Dst Where to return the dword.
10824 * @param iSegReg The index of the segment register to use for
10825 * this access. The base and limits are checked.
10826 * @param GCPtrMem The address of the guest memory.
10827 */
10828IEM_STATIC VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10829{
10830 /* The lazy approach for now... */
10831 uint32_t const *pu32Src;
10832 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10833 if (rc == VINF_SUCCESS)
10834 {
10835 *pu32Dst = *pu32Src;
10836 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
10837 }
10838 return rc;
10839}
10840
10841
10842/**
10843 * Fetches a system table qword.
10844 *
10845 * @returns Strict VBox status code.
10846 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10847 * @param pu64Dst Where to return the qword.
10848 * @param iSegReg The index of the segment register to use for
10849 * this access. The base and limits are checked.
10850 * @param GCPtrMem The address of the guest memory.
10851 */
10852IEM_STATIC VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10853{
10854 /* The lazy approach for now... */
10855 uint64_t const *pu64Src;
10856 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10857 if (rc == VINF_SUCCESS)
10858 {
10859 *pu64Dst = *pu64Src;
10860 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
10861 }
10862 return rc;
10863}
10864
10865
10866/**
10867 * Fetches a descriptor table entry with caller specified error code.
10868 *
10869 * @returns Strict VBox status code.
10870 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10871 * @param pDesc Where to return the descriptor table entry.
10872 * @param uSel The selector which table entry to fetch.
10873 * @param uXcpt The exception to raise on table lookup error.
10874 * @param uErrorCode The error code associated with the exception.
10875 */
10876IEM_STATIC VBOXSTRICTRC
10877iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt, uint16_t uErrorCode)
10878{
10879 AssertPtr(pDesc);
10880 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
10881
10882 /** @todo did the 286 require all 8 bytes to be accessible? */
10883 /*
10884 * Get the selector table base and check bounds.
10885 */
10886 RTGCPTR GCPtrBase;
10887 if (uSel & X86_SEL_LDT)
10888 {
10889 if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
10890 || (uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
10891 {
10892 Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
10893 uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel));
10894 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
10895 uErrorCode, 0);
10896 }
10897
10898 Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);
10899 GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
10900 }
10901 else
10902 {
10903 if ((uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
10904 {
10905 Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt));
10906 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
10907 uErrorCode, 0);
10908 }
10909 GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
10910 }
10911
10912 /*
10913 * Read the legacy descriptor and maybe the long mode extensions if
10914 * required.
10915 */
10916 VBOXSTRICTRC rcStrict;
10917 if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286)
10918 rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
10919 else
10920 {
10921 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0);
10922 if (rcStrict == VINF_SUCCESS)
10923 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2);
10924 if (rcStrict == VINF_SUCCESS)
10925 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4);
10926 if (rcStrict == VINF_SUCCESS)
10927 pDesc->Legacy.au16[3] = 0;
10928 else
10929 return rcStrict;
10930 }
10931
10932 if (rcStrict == VINF_SUCCESS)
10933 {
10934 if ( !IEM_IS_LONG_MODE(pVCpu)
10935 || pDesc->Legacy.Gen.u1DescType)
10936 pDesc->Long.au64[1] = 0;
10937 else if ((uint32_t)(uSel | X86_SEL_RPL_LDT) + 8 <= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt))
10938 rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel | X86_SEL_RPL_LDT) + 1);
10939 else
10940 {
10941 Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
10942 /** @todo is this the right exception? */
10943 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
10944 }
10945 }
10946 return rcStrict;
10947}
10948
10949
10950/**
10951 * Fetches a descriptor table entry.
10952 *
10953 * @returns Strict VBox status code.
10954 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10955 * @param pDesc Where to return the descriptor table entry.
10956 * @param uSel The selector which table entry to fetch.
10957 * @param uXcpt The exception to raise on table lookup error.
10958 */
10959IEM_STATIC VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt)
10960{
10961 return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
10962}
10963
10964
10965/**
10966 * Fakes a long mode stack selector for SS = 0.
10967 *
10968 * @param pDescSs Where to return the fake stack descriptor.
10969 * @param uDpl The DPL we want.
10970 */
10971IEM_STATIC void iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
10972{
10973 pDescSs->Long.au64[0] = 0;
10974 pDescSs->Long.au64[1] = 0;
10975 pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
10976 pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
10977 pDescSs->Long.Gen.u2Dpl = uDpl;
10978 pDescSs->Long.Gen.u1Present = 1;
10979 pDescSs->Long.Gen.u1Long = 1;
10980}
10981
10982
10983/**
10984 * Marks the selector descriptor as accessed (only non-system descriptors).
10985 *
10986 * This function ASSUMES that iemMemFetchSelDesc has be called previously and
10987 * will therefore skip the limit checks.
10988 *
10989 * @returns Strict VBox status code.
10990 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10991 * @param uSel The selector.
10992 */
10993IEM_STATIC VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel)
10994{
10995 /*
10996 * Get the selector table base and calculate the entry address.
10997 */
10998 RTGCPTR GCPtr = uSel & X86_SEL_LDT
10999 ? pVCpu->cpum.GstCtx.ldtr.u64Base
11000 : pVCpu->cpum.GstCtx.gdtr.pGdt;
11001 GCPtr += uSel & X86_SEL_MASK;
11002
11003 /*
11004 * ASMAtomicBitSet will assert if the address is misaligned, so do some
11005 * ugly stuff to avoid this. This will make sure it's an atomic access
11006 * as well more or less remove any question about 8-bit or 32-bit accesss.
11007 */
11008 VBOXSTRICTRC rcStrict;
11009 uint32_t volatile *pu32;
11010 if ((GCPtr & 3) == 0)
11011 {
11012 /* The normal case, map the 32-bit bits around the accessed bit (40). */
11013 GCPtr += 2 + 2;
11014 rcStrict = iemMemMap(pVCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
11015 if (rcStrict != VINF_SUCCESS)
11016 return rcStrict;
11017 ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
11018 }
11019 else
11020 {
11021 /* The misaligned GDT/LDT case, map the whole thing. */
11022 rcStrict = iemMemMap(pVCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
11023 if (rcStrict != VINF_SUCCESS)
11024 return rcStrict;
11025 switch ((uintptr_t)pu32 & 3)
11026 {
11027 case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
11028 case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
11029 case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
11030 case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
11031 }
11032 }
11033
11034 return iemMemCommitAndUnmap(pVCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
11035}
11036
11037/** @} */
11038
11039
11040/*
11041 * Include the C/C++ implementation of instruction.
11042 */
11043#include "IEMAllCImpl.cpp.h"
11044
11045
11046
11047/** @name "Microcode" macros.
11048 *
11049 * The idea is that we should be able to use the same code to interpret
11050 * instructions as well as recompiler instructions. Thus this obfuscation.
11051 *
11052 * @{
11053 */
11054#define IEM_MC_BEGIN(a_cArgs, a_cLocals) {
11055#define IEM_MC_END() }
11056#define IEM_MC_PAUSE() do {} while (0)
11057#define IEM_MC_CONTINUE() do {} while (0)
11058
11059/** Internal macro. */
11060#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
11061 do \
11062 { \
11063 VBOXSTRICTRC rcStrict2 = a_Expr; \
11064 if (rcStrict2 != VINF_SUCCESS) \
11065 return rcStrict2; \
11066 } while (0)
11067
11068
11069#define IEM_MC_ADVANCE_RIP() iemRegUpdateRipAndClearRF(pVCpu)
11070#define IEM_MC_REL_JMP_S8(a_i8) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS8(pVCpu, a_i8))
11071#define IEM_MC_REL_JMP_S16(a_i16) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS16(pVCpu, a_i16))
11072#define IEM_MC_REL_JMP_S32(a_i32) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS32(pVCpu, a_i32))
11073#define IEM_MC_SET_RIP_U16(a_u16NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u16NewIP)))
11074#define IEM_MC_SET_RIP_U32(a_u32NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u32NewIP)))
11075#define IEM_MC_SET_RIP_U64(a_u64NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u64NewIP)))
11076#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
11077#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
11078 do { \
11079 if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)) \
11080 return iemRaiseDeviceNotAvailable(pVCpu); \
11081 } while (0)
11082#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
11083 do { \
11084 if ((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP | X86_CR0_TS)) == (X86_CR0_MP | X86_CR0_TS)) \
11085 return iemRaiseDeviceNotAvailable(pVCpu); \
11086 } while (0)
11087#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
11088 do { \
11089 if (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW & X86_FSW_ES) \
11090 return iemRaiseMathFault(pVCpu); \
11091 } while (0)
11092#define IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT() \
11093 do { \
11094 if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE) \
11095 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
11096 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx2) \
11097 return iemRaiseUndefinedOpcode(pVCpu); \
11098 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11099 return iemRaiseDeviceNotAvailable(pVCpu); \
11100 } while (0)
11101#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
11102 do { \
11103 if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE) \
11104 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
11105 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx) \
11106 return iemRaiseUndefinedOpcode(pVCpu); \
11107 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11108 return iemRaiseDeviceNotAvailable(pVCpu); \
11109 } while (0)
11110#define IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT() \
11111 do { \
11112 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11113 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11114 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse41) \
11115 return iemRaiseUndefinedOpcode(pVCpu); \
11116 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11117 return iemRaiseDeviceNotAvailable(pVCpu); \
11118 } while (0)
11119#define IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT() \
11120 do { \
11121 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11122 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11123 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse3) \
11124 return iemRaiseUndefinedOpcode(pVCpu); \
11125 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11126 return iemRaiseDeviceNotAvailable(pVCpu); \
11127 } while (0)
11128#define IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT() \
11129 do { \
11130 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11131 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11132 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2) \
11133 return iemRaiseUndefinedOpcode(pVCpu); \
11134 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11135 return iemRaiseDeviceNotAvailable(pVCpu); \
11136 } while (0)
11137#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
11138 do { \
11139 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11140 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11141 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse) \
11142 return iemRaiseUndefinedOpcode(pVCpu); \
11143 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11144 return iemRaiseDeviceNotAvailable(pVCpu); \
11145 } while (0)
11146#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
11147 do { \
11148 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11149 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMmx) \
11150 return iemRaiseUndefinedOpcode(pVCpu); \
11151 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11152 return iemRaiseDeviceNotAvailable(pVCpu); \
11153 } while (0)
11154#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT() \
11155 do { \
11156 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11157 || ( !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse \
11158 && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAmdMmxExts) ) \
11159 return iemRaiseUndefinedOpcode(pVCpu); \
11160 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11161 return iemRaiseDeviceNotAvailable(pVCpu); \
11162 } while (0)
11163#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
11164 do { \
11165 if (pVCpu->iem.s.uCpl != 0) \
11166 return iemRaiseGeneralProtectionFault0(pVCpu); \
11167 } while (0)
11168#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
11169 do { \
11170 if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
11171 else return iemRaiseGeneralProtectionFault0(pVCpu); \
11172 } while (0)
11173#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
11174 do { \
11175 if ( pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT \
11176 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFsGsBase \
11177 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE)) \
11178 return iemRaiseUndefinedOpcode(pVCpu); \
11179 } while (0)
11180#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
11181 do { \
11182 if (!IEM_IS_CANONICAL(a_u64Addr)) \
11183 return iemRaiseGeneralProtectionFault0(pVCpu); \
11184 } while (0)
11185
11186
11187#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
11188#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
11189#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
11190#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
11191#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
11192#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
11193#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
11194 uint32_t a_Name; \
11195 uint32_t *a_pName = &a_Name
11196#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
11197 do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
11198
11199#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
11200#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
11201
11202#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11203#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11204#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11205#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11206#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
11207#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
11208#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
11209#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
11210#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
11211#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
11212#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
11213#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
11214#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
11215#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
11216#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
11217#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
11218#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
11219#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
11220 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11221 (a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
11222 } while (0)
11223#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
11224 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11225 (a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
11226 } while (0)
11227#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
11228 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11229 (a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
11230 } while (0)
11231/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
11232#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
11233 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11234 (a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
11235 } while (0)
11236#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
11237 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11238 (a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
11239 } while (0)
11240/** @note Not for IOPL or IF testing or modification. */
11241#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
11242#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u
11243#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW
11244#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FCW
11245
11246#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
11247#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
11248#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) /* clear high bits. */
11249#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
11250#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
11251#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
11252#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
11253#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
11254#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
11255#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
11256/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
11257#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
11258 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11259 *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
11260 } while (0)
11261#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
11262 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11263 *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); /* clear high bits. */ \
11264 } while (0)
11265#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
11266 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
11267
11268
11269#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
11270#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
11271/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
11272 * Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
11273#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
11274#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
11275/** @note Not for IOPL or IF testing or modification. */
11276#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.u
11277
11278#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) += (a_u8Value)
11279#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
11280#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
11281 do { \
11282 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11283 *pu32Reg += (a_u32Value); \
11284 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11285 } while (0)
11286#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
11287
11288#define IEM_MC_SUB_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) -= (a_u8Value)
11289#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u16Value)
11290#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
11291 do { \
11292 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11293 *pu32Reg -= (a_u32Value); \
11294 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11295 } while (0)
11296#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u64Value)
11297#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
11298
11299#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
11300#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
11301#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
11302#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
11303#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
11304#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
11305#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
11306
11307#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
11308#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
11309#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
11310#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
11311
11312#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
11313#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
11314#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
11315
11316#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) |= (a_u8Mask); } while (0)
11317#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) |= (a_u16Mask); } while (0)
11318#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) |= (a_u32Mask); } while (0)
11319
11320#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
11321#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
11322#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
11323
11324#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
11325#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
11326#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
11327
11328#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
11329
11330#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) |= (a_u32Mask); } while (0)
11331
11332#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
11333#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
11334#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
11335 do { \
11336 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11337 *pu32Reg &= (a_u32Value); \
11338 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11339 } while (0)
11340#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
11341
11342#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) |= (a_u8Value)
11343#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) |= (a_u16Value)
11344#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
11345 do { \
11346 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11347 *pu32Reg |= (a_u32Value); \
11348 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11349 } while (0)
11350#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) |= (a_u64Value)
11351
11352
11353/** @note Not for IOPL or IF modification. */
11354#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u |= (a_fBit); } while (0)
11355/** @note Not for IOPL or IF modification. */
11356#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
11357/** @note Not for IOPL or IF modification. */
11358#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
11359
11360#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW &= X86_FSW_C_MASK | X86_FSW_TOP_MASK; } while (0)
11361
11362/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
11363#define IEM_MC_FPU_TO_MMX_MODE() do { \
11364 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW &= ~X86_FSW_TOP_MASK; \
11365 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FTW = 0xff; \
11366 } while (0)
11367
11368/** Switches the FPU state from MMX mode (FTW=0xffff). */
11369#define IEM_MC_FPU_FROM_MMX_MODE() do { \
11370 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FTW = 0; \
11371 } while (0)
11372
11373#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
11374 do { (a_u64Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx; } while (0)
11375#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
11376 do { (a_u32Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].au32[0]; } while (0)
11377#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
11378 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
11379 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
11380 } while (0)
11381#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
11382 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
11383 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
11384 } while (0)
11385#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
11386 (a_pu64Dst) = (&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx)
11387#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
11388 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx)
11389#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
11390 (a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx)
11391
11392#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
11393 do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0]; \
11394 (a_u128Value).au64[1] = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1]; \
11395 } while (0)
11396#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg) \
11397 do { (a_u64Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0]; } while (0)
11398#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg) \
11399 do { (a_u32Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au32[0]; } while (0)
11400#define IEM_MC_FETCH_XREG_HI_U64(a_u64Value, a_iXReg) \
11401 do { (a_u64Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1]; } while (0)
11402#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
11403 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
11404 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
11405 } while (0)
11406#define IEM_MC_STORE_XREG_U64(a_iXReg, a_u64Value) \
11407 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); } while (0)
11408#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
11409 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
11410 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = 0; \
11411 } while (0)
11412#define IEM_MC_STORE_XREG_U32(a_iXReg, a_u32Value) \
11413 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au32[0] = (a_u32Value); } while (0)
11414#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
11415 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
11416 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = 0; \
11417 } while (0)
11418#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value) \
11419 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = (a_u64Value); } while (0)
11420#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
11421 (a_pu128Dst) = (&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].uXmm)
11422#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
11423 (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].uXmm)
11424#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
11425 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0])
11426#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
11427 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegDst)].au64[0] \
11428 = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegSrc)].au64[0]; \
11429 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegDst)].au64[1] \
11430 = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegSrc)].au64[1]; \
11431 } while (0)
11432
11433#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
11434 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11435 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11436 (a_u32Dst) = pXStateTmp->x87.aXMM[iYRegSrcTmp].au32[0]; \
11437 } while (0)
11438#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc) \
11439 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11440 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11441 (a_u64Dst) = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11442 } while (0)
11443#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
11444 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11445 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11446 (a_u128Dst).au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11447 (a_u128Dst).au64[1] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
11448 } while (0)
11449#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
11450 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11451 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11452 (a_u256Dst).au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11453 (a_u256Dst).au64[1] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
11454 (a_u256Dst).au64[2] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
11455 (a_u256Dst).au64[3] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
11456 } while (0)
11457
11458#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_pXState, a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
11459#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
11460 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11461 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11462 pXStateTmp->x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
11463 pXStateTmp->x87.aXMM[iYRegDstTmp].au32[1] = 0; \
11464 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = 0; \
11465 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11466 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11467 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11468 } while (0)
11469#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
11470 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11471 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11472 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
11473 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = 0; \
11474 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11475 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11476 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11477 } while (0)
11478#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
11479 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11480 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11481 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
11482 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
11483 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11484 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11485 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11486 } while (0)
11487#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
11488 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11489 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11490 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
11491 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
11492 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
11493 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
11494 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11495 } while (0)
11496
11497#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
11498 (a_pu128Dst) = (&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aYMM[(a_iYReg)].uXmm)
11499#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
11500 (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aYMM[(a_iYReg)].uXmm)
11501#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
11502 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aYMM[(a_iYReg)].au64[0])
11503#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
11504 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11505 uintptr_t const iYRegTmp = (a_iYReg); \
11506 pXStateTmp->u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
11507 pXStateTmp->u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
11508 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegTmp); \
11509 } while (0)
11510
11511#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
11512 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11513 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11514 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11515 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11516 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
11517 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
11518 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
11519 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11520 } while (0)
11521#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
11522 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11523 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11524 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11525 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11526 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
11527 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11528 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11529 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11530 } while (0)
11531#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
11532 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11533 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11534 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11535 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11536 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = 0; \
11537 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11538 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11539 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11540 } while (0)
11541
11542#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
11543 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11544 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11545 uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
11546 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11547 pXStateTmp->x87.aXMM[iYRegDstTmp].au32[0] = pXStateTmp->x87.aXMM[iYRegSrc32Tmp].au32[0]; \
11548 pXStateTmp->x87.aXMM[iYRegDstTmp].au32[1] = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au32[1]; \
11549 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11550 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11551 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11552 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11553 } while (0)
11554#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
11555 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11556 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11557 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
11558 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11559 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = pXStateTmp->x87.aXMM[iYRegSrc64Tmp].au64[0]; \
11560 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11561 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11562 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11563 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11564 } while (0)
11565#define IEM_MC_MERGE_YREG_U64HI_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
11566 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11567 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11568 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
11569 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11570 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = pXStateTmp->x87.aXMM[iYRegSrc64Tmp].au64[1]; \
11571 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11572 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11573 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11574 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11575 } while (0)
11576#define IEM_MC_MERGE_YREG_U64LOCAL_U64_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
11577 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11578 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11579 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11580 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
11581 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11582 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11583 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11584 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11585 } while (0)
11586
11587#ifndef IEM_WITH_SETJMP
11588# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
11589 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
11590# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
11591 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
11592# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
11593 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
11594#else
11595# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
11596 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11597# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
11598 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
11599# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
11600 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
11601#endif
11602
11603#ifndef IEM_WITH_SETJMP
11604# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11605 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
11606# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11607 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11608# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
11609 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
11610#else
11611# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11612 ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11613# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11614 ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11615# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
11616 ((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11617#endif
11618
11619#ifndef IEM_WITH_SETJMP
11620# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11621 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
11622# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11623 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11624# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
11625 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
11626#else
11627# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11628 ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11629# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11630 ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11631# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
11632 ((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11633#endif
11634
11635#ifdef SOME_UNUSED_FUNCTION
11636# define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11637 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
11638#endif
11639
11640#ifndef IEM_WITH_SETJMP
11641# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11642 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
11643# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11644 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11645# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
11646 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
11647# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
11648 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
11649#else
11650# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11651 ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11652# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11653 ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11654# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
11655 ((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11656# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
11657 ((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11658#endif
11659
11660#ifndef IEM_WITH_SETJMP
11661# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
11662 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u32, (a_iSeg), (a_GCPtrMem)))
11663# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
11664 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).au64[0], (a_iSeg), (a_GCPtrMem)))
11665# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
11666 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
11667#else
11668# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
11669 ((a_r32Dst).u32 = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11670# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
11671 ((a_r64Dst).au64[0] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11672# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
11673 iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
11674#endif
11675
11676#ifndef IEM_WITH_SETJMP
11677# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
11678 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
11679# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
11680 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
11681#else
11682# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
11683 iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
11684# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
11685 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
11686#endif
11687
11688#ifndef IEM_WITH_SETJMP
11689# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
11690 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
11691# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
11692 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
11693#else
11694# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
11695 iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
11696# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
11697 iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
11698#endif
11699
11700
11701
11702#ifndef IEM_WITH_SETJMP
11703# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11704 do { \
11705 uint8_t u8Tmp; \
11706 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11707 (a_u16Dst) = u8Tmp; \
11708 } while (0)
11709# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11710 do { \
11711 uint8_t u8Tmp; \
11712 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11713 (a_u32Dst) = u8Tmp; \
11714 } while (0)
11715# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11716 do { \
11717 uint8_t u8Tmp; \
11718 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11719 (a_u64Dst) = u8Tmp; \
11720 } while (0)
11721# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11722 do { \
11723 uint16_t u16Tmp; \
11724 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11725 (a_u32Dst) = u16Tmp; \
11726 } while (0)
11727# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11728 do { \
11729 uint16_t u16Tmp; \
11730 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11731 (a_u64Dst) = u16Tmp; \
11732 } while (0)
11733# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11734 do { \
11735 uint32_t u32Tmp; \
11736 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
11737 (a_u64Dst) = u32Tmp; \
11738 } while (0)
11739#else /* IEM_WITH_SETJMP */
11740# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11741 ((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11742# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11743 ((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11744# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11745 ((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11746# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11747 ((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11748# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11749 ((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11750# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11751 ((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11752#endif /* IEM_WITH_SETJMP */
11753
11754#ifndef IEM_WITH_SETJMP
11755# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11756 do { \
11757 uint8_t u8Tmp; \
11758 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11759 (a_u16Dst) = (int8_t)u8Tmp; \
11760 } while (0)
11761# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11762 do { \
11763 uint8_t u8Tmp; \
11764 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11765 (a_u32Dst) = (int8_t)u8Tmp; \
11766 } while (0)
11767# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11768 do { \
11769 uint8_t u8Tmp; \
11770 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11771 (a_u64Dst) = (int8_t)u8Tmp; \
11772 } while (0)
11773# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11774 do { \
11775 uint16_t u16Tmp; \
11776 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11777 (a_u32Dst) = (int16_t)u16Tmp; \
11778 } while (0)
11779# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11780 do { \
11781 uint16_t u16Tmp; \
11782 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11783 (a_u64Dst) = (int16_t)u16Tmp; \
11784 } while (0)
11785# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11786 do { \
11787 uint32_t u32Tmp; \
11788 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
11789 (a_u64Dst) = (int32_t)u32Tmp; \
11790 } while (0)
11791#else /* IEM_WITH_SETJMP */
11792# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11793 ((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11794# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11795 ((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11796# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11797 ((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11798# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11799 ((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11800# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11801 ((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11802# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11803 ((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11804#endif /* IEM_WITH_SETJMP */
11805
11806#ifndef IEM_WITH_SETJMP
11807# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
11808 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
11809# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
11810 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
11811# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
11812 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
11813# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
11814 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
11815#else
11816# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
11817 iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
11818# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
11819 iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
11820# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
11821 iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
11822# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
11823 iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
11824#endif
11825
11826#ifndef IEM_WITH_SETJMP
11827# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
11828 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
11829# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
11830 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
11831# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
11832 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
11833# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
11834 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
11835#else
11836# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
11837 iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
11838# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
11839 iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
11840# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
11841 iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
11842# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
11843 iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
11844#endif
11845
11846#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
11847#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
11848#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
11849#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
11850#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u32 = UINT32_C(0xffc00000)
11851#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->au64[0] = UINT64_C(0xfff8000000000000)
11852#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
11853 do { \
11854 (a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
11855 (a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
11856 } while (0)
11857
11858#ifndef IEM_WITH_SETJMP
11859# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
11860 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
11861# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
11862 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
11863#else
11864# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
11865 iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
11866# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
11867 iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
11868#endif
11869
11870#ifndef IEM_WITH_SETJMP
11871# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
11872 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
11873# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
11874 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
11875#else
11876# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
11877 iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
11878# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
11879 iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
11880#endif
11881
11882
11883#define IEM_MC_PUSH_U16(a_u16Value) \
11884 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
11885#define IEM_MC_PUSH_U32(a_u32Value) \
11886 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
11887#define IEM_MC_PUSH_U32_SREG(a_u32Value) \
11888 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_u32Value)))
11889#define IEM_MC_PUSH_U64(a_u64Value) \
11890 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
11891
11892#define IEM_MC_POP_U16(a_pu16Value) \
11893 IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
11894#define IEM_MC_POP_U32(a_pu32Value) \
11895 IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
11896#define IEM_MC_POP_U64(a_pu64Value) \
11897 IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
11898
11899/** Maps guest memory for direct or bounce buffered access.
11900 * The purpose is to pass it to an operand implementation, thus the a_iArg.
11901 * @remarks May return.
11902 */
11903#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
11904 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pMem), sizeof(*(a_pMem)), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
11905
11906/** Maps guest memory for direct or bounce buffered access.
11907 * The purpose is to pass it to an operand implementation, thus the a_iArg.
11908 * @remarks May return.
11909 */
11910#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_iArg) \
11911 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
11912
11913/** Commits the memory and unmaps the guest memory.
11914 * @remarks May return.
11915 */
11916#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
11917 IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess)))
11918
11919/** Commits the memory and unmaps the guest memory unless the FPU status word
11920 * indicates (@a a_u16FSW) and FPU control word indicates a pending exception
11921 * that would cause FLD not to store.
11922 *
11923 * The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
11924 * store, while \#P will not.
11925 *
11926 * @remarks May in theory return - for now.
11927 */
11928#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
11929 do { \
11930 if ( !(a_u16FSW & X86_FSW_ES) \
11931 || !( (a_u16FSW & (X86_FSW_UE | X86_FSW_OE | X86_FSW_IE)) \
11932 & ~(pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FCW & X86_FCW_MASK_ALL) ) ) \
11933 IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess))); \
11934 } while (0)
11935
11936/** Calculate efficient address from R/M. */
11937#ifndef IEM_WITH_SETJMP
11938# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
11939 IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (bRm), (cbImm), &(a_GCPtrEff)))
11940#else
11941# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
11942 ((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (bRm), (cbImm)))
11943#endif
11944
11945#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
11946#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
11947#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
11948#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
11949#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
11950#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
11951#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
11952
11953/**
11954 * Defers the rest of the instruction emulation to a C implementation routine
11955 * and returns, only taking the standard parameters.
11956 *
11957 * @param a_pfnCImpl The pointer to the C routine.
11958 * @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
11959 */
11960#define IEM_MC_CALL_CIMPL_0(a_pfnCImpl) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
11961
11962/**
11963 * Defers the rest of instruction emulation to a C implementation routine and
11964 * returns, taking one argument in addition to the standard ones.
11965 *
11966 * @param a_pfnCImpl The pointer to the C routine.
11967 * @param a0 The argument.
11968 */
11969#define IEM_MC_CALL_CIMPL_1(a_pfnCImpl, a0) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0)
11970
11971/**
11972 * Defers the rest of the instruction emulation to a C implementation routine
11973 * and returns, taking two arguments in addition to the standard ones.
11974 *
11975 * @param a_pfnCImpl The pointer to the C routine.
11976 * @param a0 The first extra argument.
11977 * @param a1 The second extra argument.
11978 */
11979#define IEM_MC_CALL_CIMPL_2(a_pfnCImpl, a0, a1) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1)
11980
11981/**
11982 * Defers the rest of the instruction emulation to a C implementation routine
11983 * and returns, taking three arguments in addition to the standard ones.
11984 *
11985 * @param a_pfnCImpl The pointer to the C routine.
11986 * @param a0 The first extra argument.
11987 * @param a1 The second extra argument.
11988 * @param a2 The third extra argument.
11989 */
11990#define IEM_MC_CALL_CIMPL_3(a_pfnCImpl, a0, a1, a2) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2)
11991
11992/**
11993 * Defers the rest of the instruction emulation to a C implementation routine
11994 * and returns, taking four arguments in addition to the standard ones.
11995 *
11996 * @param a_pfnCImpl The pointer to the C routine.
11997 * @param a0 The first extra argument.
11998 * @param a1 The second extra argument.
11999 * @param a2 The third extra argument.
12000 * @param a3 The fourth extra argument.
12001 */
12002#define IEM_MC_CALL_CIMPL_4(a_pfnCImpl, a0, a1, a2, a3) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3)
12003
12004/**
12005 * Defers the rest of the instruction emulation to a C implementation routine
12006 * and returns, taking two arguments in addition to the standard ones.
12007 *
12008 * @param a_pfnCImpl The pointer to the C routine.
12009 * @param a0 The first extra argument.
12010 * @param a1 The second extra argument.
12011 * @param a2 The third extra argument.
12012 * @param a3 The fourth extra argument.
12013 * @param a4 The fifth extra argument.
12014 */
12015#define IEM_MC_CALL_CIMPL_5(a_pfnCImpl, a0, a1, a2, a3, a4) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4)
12016
12017/**
12018 * Defers the entire instruction emulation to a C implementation routine and
12019 * returns, only taking the standard parameters.
12020 *
12021 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12022 *
12023 * @param a_pfnCImpl The pointer to the C routine.
12024 * @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
12025 */
12026#define IEM_MC_DEFER_TO_CIMPL_0(a_pfnCImpl) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
12027
12028/**
12029 * Defers the entire instruction emulation to a C implementation routine and
12030 * returns, taking one argument in addition to the standard ones.
12031 *
12032 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12033 *
12034 * @param a_pfnCImpl The pointer to the C routine.
12035 * @param a0 The argument.
12036 */
12037#define IEM_MC_DEFER_TO_CIMPL_1(a_pfnCImpl, a0) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0)
12038
12039/**
12040 * Defers the entire instruction emulation to a C implementation routine and
12041 * returns, taking two arguments in addition to the standard ones.
12042 *
12043 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12044 *
12045 * @param a_pfnCImpl The pointer to the C routine.
12046 * @param a0 The first extra argument.
12047 * @param a1 The second extra argument.
12048 */
12049#define IEM_MC_DEFER_TO_CIMPL_2(a_pfnCImpl, a0, a1) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1)
12050
12051/**
12052 * Defers the entire instruction emulation to a C implementation routine and
12053 * returns, taking three arguments in addition to the standard ones.
12054 *
12055 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12056 *
12057 * @param a_pfnCImpl The pointer to the C routine.
12058 * @param a0 The first extra argument.
12059 * @param a1 The second extra argument.
12060 * @param a2 The third extra argument.
12061 */
12062#define IEM_MC_DEFER_TO_CIMPL_3(a_pfnCImpl, a0, a1, a2) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2)
12063
12064/**
12065 * Calls a FPU assembly implementation taking one visible argument.
12066 *
12067 * @param a_pfnAImpl Pointer to the assembly FPU routine.
12068 * @param a0 The first extra argument.
12069 */
12070#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
12071 do { \
12072 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0)); \
12073 } while (0)
12074
12075/**
12076 * Calls a FPU assembly implementation taking two visible arguments.
12077 *
12078 * @param a_pfnAImpl Pointer to the assembly FPU routine.
12079 * @param a0 The first extra argument.
12080 * @param a1 The second extra argument.
12081 */
12082#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
12083 do { \
12084 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1)); \
12085 } while (0)
12086
12087/**
12088 * Calls a FPU assembly implementation taking three visible arguments.
12089 *
12090 * @param a_pfnAImpl Pointer to the assembly FPU routine.
12091 * @param a0 The first extra argument.
12092 * @param a1 The second extra argument.
12093 * @param a2 The third extra argument.
12094 */
12095#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
12096 do { \
12097 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1), (a2)); \
12098 } while (0)
12099
12100#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
12101 do { \
12102 (a_FpuData).FSW = (a_FSW); \
12103 (a_FpuData).r80Result = *(a_pr80Value); \
12104 } while (0)
12105
12106/** Pushes FPU result onto the stack. */
12107#define IEM_MC_PUSH_FPU_RESULT(a_FpuData) \
12108 iemFpuPushResult(pVCpu, &a_FpuData)
12109/** Pushes FPU result onto the stack and sets the FPUDP. */
12110#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff) \
12111 iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff)
12112
12113/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
12114#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo) \
12115 iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo)
12116
12117/** Stores FPU result in a stack register. */
12118#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg) \
12119 iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg)
12120/** Stores FPU result in a stack register and pops the stack. */
12121#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg) \
12122 iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg)
12123/** Stores FPU result in a stack register and sets the FPUDP. */
12124#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
12125 iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
12126/** Stores FPU result in a stack register, sets the FPUDP, and pops the
12127 * stack. */
12128#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
12129 iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
12130
12131/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
12132#define IEM_MC_UPDATE_FPU_OPCODE_IP() \
12133 iemFpuUpdateOpcodeAndIp(pVCpu)
12134/** Free a stack register (for FFREE and FFREEP). */
12135#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
12136 iemFpuStackFree(pVCpu, a_iStReg)
12137/** Increment the FPU stack pointer. */
12138#define IEM_MC_FPU_STACK_INC_TOP() \
12139 iemFpuStackIncTop(pVCpu)
12140/** Decrement the FPU stack pointer. */
12141#define IEM_MC_FPU_STACK_DEC_TOP() \
12142 iemFpuStackDecTop(pVCpu)
12143
12144/** Updates the FSW, FOP, FPUIP, and FPUCS. */
12145#define IEM_MC_UPDATE_FSW(a_u16FSW) \
12146 iemFpuUpdateFSW(pVCpu, a_u16FSW)
12147/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
12148#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW) \
12149 iemFpuUpdateFSW(pVCpu, a_u16FSW)
12150/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
12151#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
12152 iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
12153/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
12154#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW) \
12155 iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW)
12156/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
12157 * stack. */
12158#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
12159 iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
12160/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
12161#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW) \
12162 iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW)
12163
12164/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
12165#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst) \
12166 iemFpuStackUnderflow(pVCpu, a_iStDst)
12167/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
12168 * stack. */
12169#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst) \
12170 iemFpuStackUnderflowThenPop(pVCpu, a_iStDst)
12171/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
12172 * FPUDS. */
12173#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
12174 iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
12175/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
12176 * FPUDS. Pops stack. */
12177#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
12178 iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
12179/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
12180 * stack twice. */
12181#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP() \
12182 iemFpuStackUnderflowThenPopPop(pVCpu)
12183/** Raises a FPU stack underflow exception for an instruction pushing a result
12184 * value onto the stack. Sets FPUIP, FPUCS and FOP. */
12185#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW() \
12186 iemFpuStackPushUnderflow(pVCpu)
12187/** Raises a FPU stack underflow exception for an instruction pushing a result
12188 * value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
12189#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO() \
12190 iemFpuStackPushUnderflowTwo(pVCpu)
12191
12192/** Raises a FPU stack overflow exception as part of a push attempt. Sets
12193 * FPUIP, FPUCS and FOP. */
12194#define IEM_MC_FPU_STACK_PUSH_OVERFLOW() \
12195 iemFpuStackPushOverflow(pVCpu)
12196/** Raises a FPU stack overflow exception as part of a push attempt. Sets
12197 * FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
12198#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff) \
12199 iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff)
12200/** Prepares for using the FPU state.
12201 * Ensures that we can use the host FPU in the current context (RC+R0.
12202 * Ensures the guest FPU state in the CPUMCTX is up to date. */
12203#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
12204/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
12205#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
12206/** Actualizes the guest FPU state so it can be accessed and modified. */
12207#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
12208
12209/** Prepares for using the SSE state.
12210 * Ensures that we can use the host SSE/FPU in the current context (RC+R0.
12211 * Ensures the guest SSE state in the CPUMCTX is up to date. */
12212#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
12213/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
12214#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
12215/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
12216#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
12217
12218/** Prepares for using the AVX state.
12219 * Ensures that we can use the host AVX/FPU in the current context (RC+R0.
12220 * Ensures the guest AVX state in the CPUMCTX is up to date.
12221 * @note This will include the AVX512 state too when support for it is added
12222 * due to the zero extending feature of VEX instruction. */
12223#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
12224/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
12225#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
12226/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
12227#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
12228
12229/**
12230 * Calls a MMX assembly implementation taking two visible arguments.
12231 *
12232 * @param a_pfnAImpl Pointer to the assembly MMX routine.
12233 * @param a0 The first extra argument.
12234 * @param a1 The second extra argument.
12235 */
12236#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
12237 do { \
12238 IEM_MC_PREPARE_FPU_USAGE(); \
12239 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1)); \
12240 } while (0)
12241
12242/**
12243 * Calls a MMX assembly implementation taking three visible arguments.
12244 *
12245 * @param a_pfnAImpl Pointer to the assembly MMX routine.
12246 * @param a0 The first extra argument.
12247 * @param a1 The second extra argument.
12248 * @param a2 The third extra argument.
12249 */
12250#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
12251 do { \
12252 IEM_MC_PREPARE_FPU_USAGE(); \
12253 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1), (a2)); \
12254 } while (0)
12255
12256
12257/**
12258 * Calls a SSE assembly implementation taking two visible arguments.
12259 *
12260 * @param a_pfnAImpl Pointer to the assembly SSE routine.
12261 * @param a0 The first extra argument.
12262 * @param a1 The second extra argument.
12263 */
12264#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
12265 do { \
12266 IEM_MC_PREPARE_SSE_USAGE(); \
12267 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1)); \
12268 } while (0)
12269
12270/**
12271 * Calls a SSE assembly implementation taking three visible arguments.
12272 *
12273 * @param a_pfnAImpl Pointer to the assembly SSE routine.
12274 * @param a0 The first extra argument.
12275 * @param a1 The second extra argument.
12276 * @param a2 The third extra argument.
12277 */
12278#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
12279 do { \
12280 IEM_MC_PREPARE_SSE_USAGE(); \
12281 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1), (a2)); \
12282 } while (0)
12283
12284
12285/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
12286 * IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ... */
12287#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
12288 IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, pVCpu->cpum.GstCtx.CTX_SUFF(pXState), 0)
12289
12290/**
12291 * Calls a AVX assembly implementation taking two visible arguments.
12292 *
12293 * There is one implicit zero'th argument, a pointer to the extended state.
12294 *
12295 * @param a_pfnAImpl Pointer to the assembly AVX routine.
12296 * @param a1 The first extra argument.
12297 * @param a2 The second extra argument.
12298 */
12299#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
12300 do { \
12301 IEM_MC_PREPARE_AVX_USAGE(); \
12302 a_pfnAImpl(pXState, (a1), (a2)); \
12303 } while (0)
12304
12305/**
12306 * Calls a AVX assembly implementation taking three visible arguments.
12307 *
12308 * There is one implicit zero'th argument, a pointer to the extended state.
12309 *
12310 * @param a_pfnAImpl Pointer to the assembly AVX routine.
12311 * @param a1 The first extra argument.
12312 * @param a2 The second extra argument.
12313 * @param a3 The third extra argument.
12314 */
12315#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
12316 do { \
12317 IEM_MC_PREPARE_AVX_USAGE(); \
12318 a_pfnAImpl(pXState, (a1), (a2), (a3)); \
12319 } while (0)
12320
12321/** @note Not for IOPL or IF testing. */
12322#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
12323/** @note Not for IOPL or IF testing. */
12324#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
12325/** @note Not for IOPL or IF testing. */
12326#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
12327/** @note Not for IOPL or IF testing. */
12328#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
12329/** @note Not for IOPL or IF testing. */
12330#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
12331 if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12332 != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12333/** @note Not for IOPL or IF testing. */
12334#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
12335 if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12336 == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12337/** @note Not for IOPL or IF testing. */
12338#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
12339 if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
12340 || !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12341 != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12342/** @note Not for IOPL or IF testing. */
12343#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
12344 if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
12345 && !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12346 == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12347#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
12348#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
12349#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
12350/** @note Not for IOPL or IF testing. */
12351#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
12352 if ( pVCpu->cpum.GstCtx.cx != 0 \
12353 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12354/** @note Not for IOPL or IF testing. */
12355#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
12356 if ( pVCpu->cpum.GstCtx.ecx != 0 \
12357 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12358/** @note Not for IOPL or IF testing. */
12359#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
12360 if ( pVCpu->cpum.GstCtx.rcx != 0 \
12361 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12362/** @note Not for IOPL or IF testing. */
12363#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
12364 if ( pVCpu->cpum.GstCtx.cx != 0 \
12365 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12366/** @note Not for IOPL or IF testing. */
12367#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
12368 if ( pVCpu->cpum.GstCtx.ecx != 0 \
12369 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12370/** @note Not for IOPL or IF testing. */
12371#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
12372 if ( pVCpu->cpum.GstCtx.rcx != 0 \
12373 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12374#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
12375#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
12376
12377#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
12378 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
12379#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
12380 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
12381#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
12382 if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
12383#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
12384 if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
12385#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
12386 if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
12387#define IEM_MC_IF_FCW_IM() \
12388 if (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FCW & X86_FCW_IM) {
12389
12390#define IEM_MC_ELSE() } else {
12391#define IEM_MC_ENDIF() } do {} while (0)
12392
12393/** @} */
12394
12395
12396/** @name Opcode Debug Helpers.
12397 * @{
12398 */
12399#ifdef VBOX_WITH_STATISTICS
12400# define IEMOP_INC_STATS(a_Stats) do { pVCpu->iem.s.CTX_SUFF(pStats)->a_Stats += 1; } while (0)
12401#else
12402# define IEMOP_INC_STATS(a_Stats) do { } while (0)
12403#endif
12404
12405#ifdef DEBUG
12406# define IEMOP_MNEMONIC(a_Stats, a_szMnemonic) \
12407 do { \
12408 IEMOP_INC_STATS(a_Stats); \
12409 Log4(("decode - %04x:%RGv %s%s [#%u]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, \
12410 pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, pVCpu->iem.s.cInstructions)); \
12411 } while (0)
12412
12413# define IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
12414 do { \
12415 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12416 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12417 (void)RT_CONCAT(OP_,a_Upper); \
12418 (void)(a_fDisHints); \
12419 (void)(a_fIemHints); \
12420 } while (0)
12421
12422# define IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
12423 do { \
12424 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12425 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12426 (void)RT_CONCAT(OP_,a_Upper); \
12427 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12428 (void)(a_fDisHints); \
12429 (void)(a_fIemHints); \
12430 } while (0)
12431
12432# define IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
12433 do { \
12434 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12435 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12436 (void)RT_CONCAT(OP_,a_Upper); \
12437 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12438 (void)RT_CONCAT(OP_PARM_,a_Op2); \
12439 (void)(a_fDisHints); \
12440 (void)(a_fIemHints); \
12441 } while (0)
12442
12443# define IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
12444 do { \
12445 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12446 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12447 (void)RT_CONCAT(OP_,a_Upper); \
12448 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12449 (void)RT_CONCAT(OP_PARM_,a_Op2); \
12450 (void)RT_CONCAT(OP_PARM_,a_Op3); \
12451 (void)(a_fDisHints); \
12452 (void)(a_fIemHints); \
12453 } while (0)
12454
12455# define IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
12456 do { \
12457 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12458 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12459 (void)RT_CONCAT(OP_,a_Upper); \
12460 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12461 (void)RT_CONCAT(OP_PARM_,a_Op2); \
12462 (void)RT_CONCAT(OP_PARM_,a_Op3); \
12463 (void)RT_CONCAT(OP_PARM_,a_Op4); \
12464 (void)(a_fDisHints); \
12465 (void)(a_fIemHints); \
12466 } while (0)
12467
12468#else
12469# define IEMOP_MNEMONIC(a_Stats, a_szMnemonic) IEMOP_INC_STATS(a_Stats)
12470
12471# define IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
12472 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12473# define IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
12474 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12475# define IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
12476 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12477# define IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
12478 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12479# define IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
12480 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12481
12482#endif
12483
12484#define IEMOP_MNEMONIC0(a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
12485 IEMOP_MNEMONIC0EX(a_Lower, \
12486 #a_Lower, \
12487 a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints)
12488#define IEMOP_MNEMONIC1(a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
12489 IEMOP_MNEMONIC1EX(RT_CONCAT3(a_Lower,_,a_Op1), \
12490 #a_Lower " " #a_Op1, \
12491 a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints)
12492#define IEMOP_MNEMONIC2(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
12493 IEMOP_MNEMONIC2EX(RT_CONCAT5(a_Lower,_,a_Op1,_,a_Op2), \
12494 #a_Lower " " #a_Op1 "," #a_Op2, \
12495 a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints)
12496#define IEMOP_MNEMONIC3(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
12497 IEMOP_MNEMONIC3EX(RT_CONCAT7(a_Lower,_,a_Op1,_,a_Op2,_,a_Op3), \
12498 #a_Lower " " #a_Op1 "," #a_Op2 "," #a_Op3, \
12499 a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints)
12500#define IEMOP_MNEMONIC4(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
12501 IEMOP_MNEMONIC4EX(RT_CONCAT9(a_Lower,_,a_Op1,_,a_Op2,_,a_Op3,_,a_Op4), \
12502 #a_Lower " " #a_Op1 "," #a_Op2 "," #a_Op3 "," #a_Op4, \
12503 a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints)
12504
12505/** @} */
12506
12507
12508/** @name Opcode Helpers.
12509 * @{
12510 */
12511
12512#ifdef IN_RING3
12513# define IEMOP_HLP_MIN_CPU(a_uMinCpu, a_fOnlyIf) \
12514 do { \
12515 if (IEM_GET_TARGET_CPU(pVCpu) >= (a_uMinCpu) || !(a_fOnlyIf)) { } \
12516 else \
12517 { \
12518 (void)DBGFSTOP(pVCpu->CTX_SUFF(pVM)); \
12519 return IEMOP_RAISE_INVALID_OPCODE(); \
12520 } \
12521 } while (0)
12522#else
12523# define IEMOP_HLP_MIN_CPU(a_uMinCpu, a_fOnlyIf) \
12524 do { \
12525 if (IEM_GET_TARGET_CPU(pVCpu) >= (a_uMinCpu) || !(a_fOnlyIf)) { } \
12526 else return IEMOP_RAISE_INVALID_OPCODE(); \
12527 } while (0)
12528#endif
12529
12530/** The instruction requires a 186 or later. */
12531#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_186
12532# define IEMOP_HLP_MIN_186() do { } while (0)
12533#else
12534# define IEMOP_HLP_MIN_186() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_186, true)
12535#endif
12536
12537/** The instruction requires a 286 or later. */
12538#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_286
12539# define IEMOP_HLP_MIN_286() do { } while (0)
12540#else
12541# define IEMOP_HLP_MIN_286() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_286, true)
12542#endif
12543
12544/** The instruction requires a 386 or later. */
12545#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_386
12546# define IEMOP_HLP_MIN_386() do { } while (0)
12547#else
12548# define IEMOP_HLP_MIN_386() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_386, true)
12549#endif
12550
12551/** The instruction requires a 386 or later if the given expression is true. */
12552#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_386
12553# define IEMOP_HLP_MIN_386_EX(a_fOnlyIf) do { } while (0)
12554#else
12555# define IEMOP_HLP_MIN_386_EX(a_fOnlyIf) IEMOP_HLP_MIN_CPU(IEMTARGETCPU_386, a_fOnlyIf)
12556#endif
12557
12558/** The instruction requires a 486 or later. */
12559#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_486
12560# define IEMOP_HLP_MIN_486() do { } while (0)
12561#else
12562# define IEMOP_HLP_MIN_486() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_486, true)
12563#endif
12564
12565/** The instruction requires a Pentium (586) or later. */
12566#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_PENTIUM
12567# define IEMOP_HLP_MIN_586() do { } while (0)
12568#else
12569# define IEMOP_HLP_MIN_586() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_PENTIUM, true)
12570#endif
12571
12572/** The instruction requires a PentiumPro (686) or later. */
12573#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_PPRO
12574# define IEMOP_HLP_MIN_686() do { } while (0)
12575#else
12576# define IEMOP_HLP_MIN_686() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_PPRO, true)
12577#endif
12578
12579
12580/** The instruction raises an \#UD in real and V8086 mode. */
12581#define IEMOP_HLP_NO_REAL_OR_V86_MODE() \
12582 do \
12583 { \
12584 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu)) { /* likely */ } \
12585 else return IEMOP_RAISE_INVALID_OPCODE(); \
12586 } while (0)
12587
12588#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12589/** This instruction raises an \#UD in real and V8086 mode or when not using a
12590 * 64-bit code segment when in long mode (applicable to all VMX instructions
12591 * except VMCALL).
12592 */
12593#define IEMOP_HLP_VMX_INSTR(a_szInstr, a_InsDiagPrefix) \
12594 do \
12595 { \
12596 if ( !IEM_IS_REAL_OR_V86_MODE(pVCpu) \
12597 && ( !IEM_IS_LONG_MODE(pVCpu) \
12598 || IEM_IS_64BIT_CODE(pVCpu))) \
12599 { /* likely */ } \
12600 else \
12601 { \
12602 if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) \
12603 { \
12604 pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_RealOrV86Mode; \
12605 Log5((a_szInstr ": Real or v8086 mode -> #UD\n")); \
12606 return IEMOP_RAISE_INVALID_OPCODE(); \
12607 } \
12608 if (IEM_IS_LONG_MODE(pVCpu) && !IEM_IS_64BIT_CODE(pVCpu)) \
12609 { \
12610 pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_LongModeCS; \
12611 Log5((a_szInstr ": Long mode without 64-bit code segment -> #UD\n")); \
12612 return IEMOP_RAISE_INVALID_OPCODE(); \
12613 } \
12614 } \
12615 } while (0)
12616
12617/** The instruction can only be executed in VMX operation (VMX root mode and
12618 * non-root mode).
12619 *
12620 * @note Update IEM_VMX_IN_VMX_OPERATION if changes are made here.
12621 */
12622# define IEMOP_HLP_IN_VMX_OPERATION(a_szInstr, a_InsDiagPrefix) \
12623 do \
12624 { \
12625 if (IEM_VMX_IS_ROOT_MODE(pVCpu)) { /* likely */ } \
12626 else \
12627 { \
12628 pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_VmxRoot; \
12629 Log5((a_szInstr ": Not in VMX operation (root mode) -> #UD\n")); \
12630 return IEMOP_RAISE_INVALID_OPCODE(); \
12631 } \
12632 } while (0)
12633#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12634
12635/** The instruction is not available in 64-bit mode, throw \#UD if we're in
12636 * 64-bit mode. */
12637#define IEMOP_HLP_NO_64BIT() \
12638 do \
12639 { \
12640 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
12641 return IEMOP_RAISE_INVALID_OPCODE(); \
12642 } while (0)
12643
12644/** The instruction is only available in 64-bit mode, throw \#UD if we're not in
12645 * 64-bit mode. */
12646#define IEMOP_HLP_ONLY_64BIT() \
12647 do \
12648 { \
12649 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) \
12650 return IEMOP_RAISE_INVALID_OPCODE(); \
12651 } while (0)
12652
12653/** The instruction defaults to 64-bit operand size if 64-bit mode. */
12654#define IEMOP_HLP_DEFAULT_64BIT_OP_SIZE() \
12655 do \
12656 { \
12657 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
12658 iemRecalEffOpSize64Default(pVCpu); \
12659 } while (0)
12660
12661/** The instruction has 64-bit operand size if 64-bit mode. */
12662#define IEMOP_HLP_64BIT_OP_SIZE() \
12663 do \
12664 { \
12665 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
12666 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; \
12667 } while (0)
12668
12669/** Only a REX prefix immediately preceeding the first opcode byte takes
12670 * effect. This macro helps ensuring this as well as logging bad guest code. */
12671#define IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE(a_szPrf) \
12672 do \
12673 { \
12674 if (RT_UNLIKELY(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX)) \
12675 { \
12676 Log5((a_szPrf ": Overriding REX prefix at %RX16! fPrefixes=%#x\n", pVCpu->cpum.GstCtx.rip, pVCpu->iem.s.fPrefixes)); \
12677 pVCpu->iem.s.fPrefixes &= ~IEM_OP_PRF_REX_MASK; \
12678 pVCpu->iem.s.uRexB = 0; \
12679 pVCpu->iem.s.uRexIndex = 0; \
12680 pVCpu->iem.s.uRexReg = 0; \
12681 iemRecalEffOpSize(pVCpu); \
12682 } \
12683 } while (0)
12684
12685/**
12686 * Done decoding.
12687 */
12688#define IEMOP_HLP_DONE_DECODING() \
12689 do \
12690 { \
12691 /*nothing for now, maybe later... */ \
12692 } while (0)
12693
12694/**
12695 * Done decoding, raise \#UD exception if lock prefix present.
12696 */
12697#define IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX() \
12698 do \
12699 { \
12700 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
12701 { /* likely */ } \
12702 else \
12703 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12704 } while (0)
12705
12706
12707/**
12708 * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
12709 * repnz or size prefixes are present, or if in real or v8086 mode.
12710 */
12711#define IEMOP_HLP_DONE_VEX_DECODING() \
12712 do \
12713 { \
12714 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12715 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
12716 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )) \
12717 { /* likely */ } \
12718 else \
12719 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12720 } while (0)
12721
12722/**
12723 * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
12724 * repnz or size prefixes are present, or if in real or v8086 mode.
12725 */
12726#define IEMOP_HLP_DONE_VEX_DECODING_L0() \
12727 do \
12728 { \
12729 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12730 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
12731 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) \
12732 && pVCpu->iem.s.uVexLength == 0)) \
12733 { /* likely */ } \
12734 else \
12735 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12736 } while (0)
12737
12738
12739/**
12740 * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
12741 * repnz or size prefixes are present, or if the VEX.VVVV field doesn't indicate
12742 * register 0, or if in real or v8086 mode.
12743 */
12744#define IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV() \
12745 do \
12746 { \
12747 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12748 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
12749 && !pVCpu->iem.s.uVex3rdReg \
12750 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )) \
12751 { /* likely */ } \
12752 else \
12753 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12754 } while (0)
12755
12756/**
12757 * Done decoding VEX, no V, L=0.
12758 * Raises \#UD exception if rex, rep, opsize or lock prefixes are present, if
12759 * we're in real or v8086 mode, if VEX.V!=0xf, or if VEX.L!=0.
12760 */
12761#define IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV() \
12762 do \
12763 { \
12764 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12765 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REX)) \
12766 && pVCpu->iem.s.uVexLength == 0 \
12767 && pVCpu->iem.s.uVex3rdReg == 0 \
12768 && !IEM_IS_REAL_OR_V86_MODE(pVCpu))) \
12769 { /* likely */ } \
12770 else \
12771 return IEMOP_RAISE_INVALID_OPCODE(); \
12772 } while (0)
12773
12774#define IEMOP_HLP_DECODED_NL_1(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_fDisOpType) \
12775 do \
12776 { \
12777 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
12778 { /* likely */ } \
12779 else \
12780 { \
12781 NOREF(a_uDisOpNo); NOREF(a_fIemOpFlags); NOREF(a_uDisParam0); NOREF(a_fDisOpType); \
12782 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12783 } \
12784 } while (0)
12785#define IEMOP_HLP_DECODED_NL_2(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_uDisParam1, a_fDisOpType) \
12786 do \
12787 { \
12788 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
12789 { /* likely */ } \
12790 else \
12791 { \
12792 NOREF(a_uDisOpNo); NOREF(a_fIemOpFlags); NOREF(a_uDisParam0); NOREF(a_uDisParam1); NOREF(a_fDisOpType); \
12793 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12794 } \
12795 } while (0)
12796
12797/**
12798 * Done decoding, raise \#UD exception if any lock, repz or repnz prefixes
12799 * are present.
12800 */
12801#define IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES() \
12802 do \
12803 { \
12804 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)))) \
12805 { /* likely */ } \
12806 else \
12807 return IEMOP_RAISE_INVALID_OPCODE(); \
12808 } while (0)
12809
12810/**
12811 * Done decoding, raise \#UD exception if any operand-size override, repz or repnz
12812 * prefixes are present.
12813 */
12814#define IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES() \
12815 do \
12816 { \
12817 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)))) \
12818 { /* likely */ } \
12819 else \
12820 return IEMOP_RAISE_INVALID_OPCODE(); \
12821 } while (0)
12822
12823
12824/**
12825 * Calculates the effective address of a ModR/M memory operand.
12826 *
12827 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
12828 *
12829 * @return Strict VBox status code.
12830 * @param pVCpu The cross context virtual CPU structure of the calling thread.
12831 * @param bRm The ModRM byte.
12832 * @param cbImm The size of any immediate following the
12833 * effective address opcode bytes. Important for
12834 * RIP relative addressing.
12835 * @param pGCPtrEff Where to return the effective address.
12836 */
12837IEM_STATIC VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff)
12838{
12839 Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
12840# define SET_SS_DEF() \
12841 do \
12842 { \
12843 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
12844 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
12845 } while (0)
12846
12847 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
12848 {
12849/** @todo Check the effective address size crap! */
12850 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
12851 {
12852 uint16_t u16EffAddr;
12853
12854 /* Handle the disp16 form with no registers first. */
12855 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
12856 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
12857 else
12858 {
12859 /* Get the displacment. */
12860 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
12861 {
12862 case 0: u16EffAddr = 0; break;
12863 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
12864 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
12865 default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
12866 }
12867
12868 /* Add the base and index registers to the disp. */
12869 switch (bRm & X86_MODRM_RM_MASK)
12870 {
12871 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
12872 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
12873 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
12874 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
12875 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
12876 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
12877 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
12878 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
12879 }
12880 }
12881
12882 *pGCPtrEff = u16EffAddr;
12883 }
12884 else
12885 {
12886 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
12887 uint32_t u32EffAddr;
12888
12889 /* Handle the disp32 form with no registers first. */
12890 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
12891 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
12892 else
12893 {
12894 /* Get the register (or SIB) value. */
12895 switch ((bRm & X86_MODRM_RM_MASK))
12896 {
12897 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
12898 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
12899 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
12900 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
12901 case 4: /* SIB */
12902 {
12903 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
12904
12905 /* Get the index and scale it. */
12906 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
12907 {
12908 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
12909 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
12910 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
12911 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
12912 case 4: u32EffAddr = 0; /*none */ break;
12913 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
12914 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
12915 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
12916 IEM_NOT_REACHED_DEFAULT_CASE_RET();
12917 }
12918 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
12919
12920 /* add base */
12921 switch (bSib & X86_SIB_BASE_MASK)
12922 {
12923 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
12924 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
12925 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
12926 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
12927 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
12928 case 5:
12929 if ((bRm & X86_MODRM_MOD_MASK) != 0)
12930 {
12931 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
12932 SET_SS_DEF();
12933 }
12934 else
12935 {
12936 uint32_t u32Disp;
12937 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
12938 u32EffAddr += u32Disp;
12939 }
12940 break;
12941 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
12942 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
12943 IEM_NOT_REACHED_DEFAULT_CASE_RET();
12944 }
12945 break;
12946 }
12947 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
12948 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
12949 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
12950 IEM_NOT_REACHED_DEFAULT_CASE_RET();
12951 }
12952
12953 /* Get and add the displacement. */
12954 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
12955 {
12956 case 0:
12957 break;
12958 case 1:
12959 {
12960 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
12961 u32EffAddr += i8Disp;
12962 break;
12963 }
12964 case 2:
12965 {
12966 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
12967 u32EffAddr += u32Disp;
12968 break;
12969 }
12970 default:
12971 AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
12972 }
12973
12974 }
12975 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
12976 *pGCPtrEff = u32EffAddr;
12977 else
12978 {
12979 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
12980 *pGCPtrEff = u32EffAddr & UINT16_MAX;
12981 }
12982 }
12983 }
12984 else
12985 {
12986 uint64_t u64EffAddr;
12987
12988 /* Handle the rip+disp32 form with no registers first. */
12989 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
12990 {
12991 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
12992 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
12993 }
12994 else
12995 {
12996 /* Get the register (or SIB) value. */
12997 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
12998 {
12999 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13000 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13001 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13002 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13003 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
13004 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13005 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13006 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13007 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13008 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13009 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13010 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13011 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13012 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13013 /* SIB */
13014 case 4:
13015 case 12:
13016 {
13017 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13018
13019 /* Get the index and scale it. */
13020 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
13021 {
13022 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13023 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13024 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13025 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13026 case 4: u64EffAddr = 0; /*none */ break;
13027 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
13028 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13029 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13030 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13031 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13032 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13033 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13034 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
13035 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13036 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13037 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13038 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13039 }
13040 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13041
13042 /* add base */
13043 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
13044 {
13045 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
13046 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
13047 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
13048 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
13049 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
13050 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
13051 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
13052 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
13053 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
13054 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
13055 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
13056 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
13057 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
13058 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
13059 /* complicated encodings */
13060 case 5:
13061 case 13:
13062 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13063 {
13064 if (!pVCpu->iem.s.uRexB)
13065 {
13066 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
13067 SET_SS_DEF();
13068 }
13069 else
13070 u64EffAddr += pVCpu->cpum.GstCtx.r13;
13071 }
13072 else
13073 {
13074 uint32_t u32Disp;
13075 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13076 u64EffAddr += (int32_t)u32Disp;
13077 }
13078 break;
13079 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13080 }
13081 break;
13082 }
13083 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13084 }
13085
13086 /* Get and add the displacement. */
13087 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13088 {
13089 case 0:
13090 break;
13091 case 1:
13092 {
13093 int8_t i8Disp;
13094 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13095 u64EffAddr += i8Disp;
13096 break;
13097 }
13098 case 2:
13099 {
13100 uint32_t u32Disp;
13101 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13102 u64EffAddr += (int32_t)u32Disp;
13103 break;
13104 }
13105 IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
13106 }
13107
13108 }
13109
13110 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
13111 *pGCPtrEff = u64EffAddr;
13112 else
13113 {
13114 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13115 *pGCPtrEff = u64EffAddr & UINT32_MAX;
13116 }
13117 }
13118
13119 Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
13120 return VINF_SUCCESS;
13121}
13122
13123
13124/**
13125 * Calculates the effective address of a ModR/M memory operand.
13126 *
13127 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
13128 *
13129 * @return Strict VBox status code.
13130 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13131 * @param bRm The ModRM byte.
13132 * @param cbImm The size of any immediate following the
13133 * effective address opcode bytes. Important for
13134 * RIP relative addressing.
13135 * @param pGCPtrEff Where to return the effective address.
13136 * @param offRsp RSP displacement.
13137 */
13138IEM_STATIC VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff, int8_t offRsp)
13139{
13140 Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
13141# define SET_SS_DEF() \
13142 do \
13143 { \
13144 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
13145 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
13146 } while (0)
13147
13148 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
13149 {
13150/** @todo Check the effective address size crap! */
13151 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
13152 {
13153 uint16_t u16EffAddr;
13154
13155 /* Handle the disp16 form with no registers first. */
13156 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
13157 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
13158 else
13159 {
13160 /* Get the displacment. */
13161 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13162 {
13163 case 0: u16EffAddr = 0; break;
13164 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
13165 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
13166 default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
13167 }
13168
13169 /* Add the base and index registers to the disp. */
13170 switch (bRm & X86_MODRM_RM_MASK)
13171 {
13172 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
13173 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
13174 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
13175 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
13176 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
13177 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
13178 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
13179 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
13180 }
13181 }
13182
13183 *pGCPtrEff = u16EffAddr;
13184 }
13185 else
13186 {
13187 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13188 uint32_t u32EffAddr;
13189
13190 /* Handle the disp32 form with no registers first. */
13191 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13192 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
13193 else
13194 {
13195 /* Get the register (or SIB) value. */
13196 switch ((bRm & X86_MODRM_RM_MASK))
13197 {
13198 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13199 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13200 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13201 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13202 case 4: /* SIB */
13203 {
13204 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13205
13206 /* Get the index and scale it. */
13207 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
13208 {
13209 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13210 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13211 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13212 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13213 case 4: u32EffAddr = 0; /*none */ break;
13214 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
13215 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13216 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13217 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13218 }
13219 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13220
13221 /* add base */
13222 switch (bSib & X86_SIB_BASE_MASK)
13223 {
13224 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
13225 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
13226 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
13227 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
13228 case 4:
13229 u32EffAddr += pVCpu->cpum.GstCtx.esp + offRsp;
13230 SET_SS_DEF();
13231 break;
13232 case 5:
13233 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13234 {
13235 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
13236 SET_SS_DEF();
13237 }
13238 else
13239 {
13240 uint32_t u32Disp;
13241 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13242 u32EffAddr += u32Disp;
13243 }
13244 break;
13245 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
13246 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
13247 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13248 }
13249 break;
13250 }
13251 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
13252 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13253 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13254 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13255 }
13256
13257 /* Get and add the displacement. */
13258 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13259 {
13260 case 0:
13261 break;
13262 case 1:
13263 {
13264 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13265 u32EffAddr += i8Disp;
13266 break;
13267 }
13268 case 2:
13269 {
13270 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13271 u32EffAddr += u32Disp;
13272 break;
13273 }
13274 default:
13275 AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
13276 }
13277
13278 }
13279 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
13280 *pGCPtrEff = u32EffAddr;
13281 else
13282 {
13283 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
13284 *pGCPtrEff = u32EffAddr & UINT16_MAX;
13285 }
13286 }
13287 }
13288 else
13289 {
13290 uint64_t u64EffAddr;
13291
13292 /* Handle the rip+disp32 form with no registers first. */
13293 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13294 {
13295 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
13296 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
13297 }
13298 else
13299 {
13300 /* Get the register (or SIB) value. */
13301 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
13302 {
13303 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13304 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13305 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13306 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13307 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
13308 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13309 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13310 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13311 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13312 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13313 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13314 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13315 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13316 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13317 /* SIB */
13318 case 4:
13319 case 12:
13320 {
13321 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13322
13323 /* Get the index and scale it. */
13324 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
13325 {
13326 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13327 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13328 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13329 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13330 case 4: u64EffAddr = 0; /*none */ break;
13331 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
13332 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13333 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13334 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13335 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13336 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13337 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13338 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
13339 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13340 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13341 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13342 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13343 }
13344 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13345
13346 /* add base */
13347 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
13348 {
13349 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
13350 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
13351 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
13352 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
13353 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + offRsp; SET_SS_DEF(); break;
13354 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
13355 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
13356 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
13357 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
13358 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
13359 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
13360 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
13361 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
13362 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
13363 /* complicated encodings */
13364 case 5:
13365 case 13:
13366 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13367 {
13368 if (!pVCpu->iem.s.uRexB)
13369 {
13370 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
13371 SET_SS_DEF();
13372 }
13373 else
13374 u64EffAddr += pVCpu->cpum.GstCtx.r13;
13375 }
13376 else
13377 {
13378 uint32_t u32Disp;
13379 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13380 u64EffAddr += (int32_t)u32Disp;
13381 }
13382 break;
13383 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13384 }
13385 break;
13386 }
13387 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13388 }
13389
13390 /* Get and add the displacement. */
13391 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13392 {
13393 case 0:
13394 break;
13395 case 1:
13396 {
13397 int8_t i8Disp;
13398 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13399 u64EffAddr += i8Disp;
13400 break;
13401 }
13402 case 2:
13403 {
13404 uint32_t u32Disp;
13405 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13406 u64EffAddr += (int32_t)u32Disp;
13407 break;
13408 }
13409 IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
13410 }
13411
13412 }
13413
13414 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
13415 *pGCPtrEff = u64EffAddr;
13416 else
13417 {
13418 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13419 *pGCPtrEff = u64EffAddr & UINT32_MAX;
13420 }
13421 }
13422
13423 Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
13424 return VINF_SUCCESS;
13425}
13426
13427
13428#ifdef IEM_WITH_SETJMP
13429/**
13430 * Calculates the effective address of a ModR/M memory operand.
13431 *
13432 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
13433 *
13434 * May longjmp on internal error.
13435 *
13436 * @return The effective address.
13437 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13438 * @param bRm The ModRM byte.
13439 * @param cbImm The size of any immediate following the
13440 * effective address opcode bytes. Important for
13441 * RIP relative addressing.
13442 */
13443IEM_STATIC RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm)
13444{
13445 Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
13446# define SET_SS_DEF() \
13447 do \
13448 { \
13449 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
13450 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
13451 } while (0)
13452
13453 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
13454 {
13455/** @todo Check the effective address size crap! */
13456 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
13457 {
13458 uint16_t u16EffAddr;
13459
13460 /* Handle the disp16 form with no registers first. */
13461 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
13462 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
13463 else
13464 {
13465 /* Get the displacment. */
13466 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13467 {
13468 case 0: u16EffAddr = 0; break;
13469 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
13470 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
13471 default: AssertFailedStmt(longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_1)); /* (caller checked for these) */
13472 }
13473
13474 /* Add the base and index registers to the disp. */
13475 switch (bRm & X86_MODRM_RM_MASK)
13476 {
13477 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
13478 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
13479 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
13480 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
13481 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
13482 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
13483 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
13484 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
13485 }
13486 }
13487
13488 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr));
13489 return u16EffAddr;
13490 }
13491
13492 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13493 uint32_t u32EffAddr;
13494
13495 /* Handle the disp32 form with no registers first. */
13496 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13497 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
13498 else
13499 {
13500 /* Get the register (or SIB) value. */
13501 switch ((bRm & X86_MODRM_RM_MASK))
13502 {
13503 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13504 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13505 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13506 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13507 case 4: /* SIB */
13508 {
13509 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13510
13511 /* Get the index and scale it. */
13512 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
13513 {
13514 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13515 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13516 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13517 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13518 case 4: u32EffAddr = 0; /*none */ break;
13519 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
13520 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13521 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13522 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13523 }
13524 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13525
13526 /* add base */
13527 switch (bSib & X86_SIB_BASE_MASK)
13528 {
13529 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
13530 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
13531 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
13532 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
13533 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
13534 case 5:
13535 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13536 {
13537 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
13538 SET_SS_DEF();
13539 }
13540 else
13541 {
13542 uint32_t u32Disp;
13543 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13544 u32EffAddr += u32Disp;
13545 }
13546 break;
13547 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
13548 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
13549 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13550 }
13551 break;
13552 }
13553 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
13554 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13555 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13556 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13557 }
13558
13559 /* Get and add the displacement. */
13560 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13561 {
13562 case 0:
13563 break;
13564 case 1:
13565 {
13566 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13567 u32EffAddr += i8Disp;
13568 break;
13569 }
13570 case 2:
13571 {
13572 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13573 u32EffAddr += u32Disp;
13574 break;
13575 }
13576 default:
13577 AssertFailedStmt(longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_2)); /* (caller checked for these) */
13578 }
13579 }
13580
13581 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
13582 {
13583 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr));
13584 return u32EffAddr;
13585 }
13586 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
13587 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX32\n", u32EffAddr & UINT16_MAX));
13588 return u32EffAddr & UINT16_MAX;
13589 }
13590
13591 uint64_t u64EffAddr;
13592
13593 /* Handle the rip+disp32 form with no registers first. */
13594 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13595 {
13596 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
13597 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
13598 }
13599 else
13600 {
13601 /* Get the register (or SIB) value. */
13602 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
13603 {
13604 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13605 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13606 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13607 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13608 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
13609 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13610 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13611 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13612 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13613 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13614 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13615 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13616 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13617 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13618 /* SIB */
13619 case 4:
13620 case 12:
13621 {
13622 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13623
13624 /* Get the index and scale it. */
13625 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
13626 {
13627 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13628 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13629 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13630 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13631 case 4: u64EffAddr = 0; /*none */ break;
13632 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
13633 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13634 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13635 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13636 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13637 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13638 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13639 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
13640 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13641 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13642 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13643 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13644 }
13645 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13646
13647 /* add base */
13648 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
13649 {
13650 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
13651 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
13652 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
13653 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
13654 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
13655 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
13656 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
13657 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
13658 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
13659 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
13660 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
13661 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
13662 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
13663 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
13664 /* complicated encodings */
13665 case 5:
13666 case 13:
13667 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13668 {
13669 if (!pVCpu->iem.s.uRexB)
13670 {
13671 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
13672 SET_SS_DEF();
13673 }
13674 else
13675 u64EffAddr += pVCpu->cpum.GstCtx.r13;
13676 }
13677 else
13678 {
13679 uint32_t u32Disp;
13680 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13681 u64EffAddr += (int32_t)u32Disp;
13682 }
13683 break;
13684 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13685 }
13686 break;
13687 }
13688 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13689 }
13690
13691 /* Get and add the displacement. */
13692 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13693 {
13694 case 0:
13695 break;
13696 case 1:
13697 {
13698 int8_t i8Disp;
13699 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13700 u64EffAddr += i8Disp;
13701 break;
13702 }
13703 case 2:
13704 {
13705 uint32_t u32Disp;
13706 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13707 u64EffAddr += (int32_t)u32Disp;
13708 break;
13709 }
13710 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
13711 }
13712
13713 }
13714
13715 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
13716 {
13717 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr));
13718 return u64EffAddr;
13719 }
13720 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13721 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX));
13722 return u64EffAddr & UINT32_MAX;
13723}
13724#endif /* IEM_WITH_SETJMP */
13725
13726/** @} */
13727
13728
13729
13730/*
13731 * Include the instructions
13732 */
13733#include "IEMAllInstructions.cpp.h"
13734
13735
13736
13737#ifdef LOG_ENABLED
13738/**
13739 * Logs the current instruction.
13740 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
13741 * @param fSameCtx Set if we have the same context information as the VMM,
13742 * clear if we may have already executed an instruction in
13743 * our debug context. When clear, we assume IEMCPU holds
13744 * valid CPU mode info.
13745 *
13746 * The @a fSameCtx parameter is now misleading and obsolete.
13747 * @param pszFunction The IEM function doing the execution.
13748 */
13749IEM_STATIC void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction)
13750{
13751# ifdef IN_RING3
13752 if (LogIs2Enabled())
13753 {
13754 char szInstr[256];
13755 uint32_t cbInstr = 0;
13756 if (fSameCtx)
13757 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
13758 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
13759 szInstr, sizeof(szInstr), &cbInstr);
13760 else
13761 {
13762 uint32_t fFlags = 0;
13763 switch (pVCpu->iem.s.enmCpuMode)
13764 {
13765 case IEMMODE_64BIT: fFlags |= DBGF_DISAS_FLAGS_64BIT_MODE; break;
13766 case IEMMODE_32BIT: fFlags |= DBGF_DISAS_FLAGS_32BIT_MODE; break;
13767 case IEMMODE_16BIT:
13768 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) || pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
13769 fFlags |= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
13770 else
13771 fFlags |= DBGF_DISAS_FLAGS_16BIT_MODE;
13772 break;
13773 }
13774 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,
13775 szInstr, sizeof(szInstr), &cbInstr);
13776 }
13777
13778 PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
13779 Log2(("**** %s\n"
13780 " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
13781 " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
13782 " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
13783 " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
13784 " %s\n"
13785 , pszFunction,
13786 pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
13787 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
13788 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
13789 pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
13790 pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
13791 szInstr));
13792
13793 if (LogIs3Enabled())
13794 DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
13795 }
13796 else
13797# endif
13798 LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
13799 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
13800 RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);
13801}
13802#endif /* LOG_ENABLED */
13803
13804
13805#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13806/**
13807 * Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW,
13808 * VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW.
13809 *
13810 * @returns Modified rcStrict.
13811 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13812 * @param rcStrict The instruction execution status.
13813 */
13814static VBOXSTRICTRC iemHandleNestedInstructionBoundraryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
13815{
13816 Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)));
13817 if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF))
13818 {
13819 /* VMX preemption timer takes priority over NMI-window exits. */
13820 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
13821 {
13822 rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
13823 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
13824 }
13825 /*
13826 * Check remaining intercepts.
13827 *
13828 * NMI-window and Interrupt-window VM-exits.
13829 * Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs.
13830 * Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits.
13831 *
13832 * See Intel spec. 26.7.6 "NMI-Window Exiting".
13833 * See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
13834 */
13835 else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW | VMCPU_FF_VMX_INT_WINDOW)
13836 && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13837 && !TRPMHasTrap(pVCpu))
13838 {
13839 Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
13840 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
13841 && CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))
13842 {
13843 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
13844 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
13845 }
13846 else if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
13847 && CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
13848 {
13849 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
13850 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
13851 }
13852 }
13853 }
13854 /* TPR-below threshold/APIC write has the highest priority. */
13855 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
13856 {
13857 rcStrict = iemVmxApicWriteEmulation(pVCpu);
13858 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13859 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
13860 }
13861 /* MTF takes priority over VMX-preemption timer. */
13862 else
13863 {
13864 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */);
13865 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13866 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
13867 }
13868 return rcStrict;
13869}
13870#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13871
13872
13873/**
13874 * Makes status code addjustments (pass up from I/O and access handler)
13875 * as well as maintaining statistics.
13876 *
13877 * @returns Strict VBox status code to pass up.
13878 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13879 * @param rcStrict The status from executing an instruction.
13880 */
13881DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
13882{
13883 if (rcStrict != VINF_SUCCESS)
13884 {
13885 if (RT_SUCCESS(rcStrict))
13886 {
13887 AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
13888 || rcStrict == VINF_IOM_R3_IOPORT_READ
13889 || rcStrict == VINF_IOM_R3_IOPORT_WRITE
13890 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
13891 || rcStrict == VINF_IOM_R3_MMIO_READ
13892 || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
13893 || rcStrict == VINF_IOM_R3_MMIO_WRITE
13894 || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
13895 || rcStrict == VINF_CPUM_R3_MSR_READ
13896 || rcStrict == VINF_CPUM_R3_MSR_WRITE
13897 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
13898 || rcStrict == VINF_EM_RAW_TO_R3
13899 || rcStrict == VINF_EM_TRIPLE_FAULT
13900 || rcStrict == VINF_GIM_R3_HYPERCALL
13901 /* raw-mode / virt handlers only: */
13902 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
13903 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
13904 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
13905 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
13906 || rcStrict == VINF_SELM_SYNC_GDT
13907 || rcStrict == VINF_CSAM_PENDING_ACTION
13908 || rcStrict == VINF_PATM_CHECK_PATCH_PAGE
13909 /* nested hw.virt codes: */
13910 || rcStrict == VINF_VMX_VMEXIT
13911 || rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
13912 || rcStrict == VINF_SVM_VMEXIT
13913 , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13914/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
13915 int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
13916#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13917 if ( rcStrict == VINF_VMX_VMEXIT
13918 && rcPassUp == VINF_SUCCESS)
13919 rcStrict = VINF_SUCCESS;
13920 else
13921#endif
13922#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
13923 if ( rcStrict == VINF_SVM_VMEXIT
13924 && rcPassUp == VINF_SUCCESS)
13925 rcStrict = VINF_SUCCESS;
13926 else
13927#endif
13928 if (rcPassUp == VINF_SUCCESS)
13929 pVCpu->iem.s.cRetInfStatuses++;
13930 else if ( rcPassUp < VINF_EM_FIRST
13931 || rcPassUp > VINF_EM_LAST
13932 || rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
13933 {
13934 Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
13935 pVCpu->iem.s.cRetPassUpStatus++;
13936 rcStrict = rcPassUp;
13937 }
13938 else
13939 {
13940 Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
13941 pVCpu->iem.s.cRetInfStatuses++;
13942 }
13943 }
13944 else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
13945 pVCpu->iem.s.cRetAspectNotImplemented++;
13946 else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
13947 pVCpu->iem.s.cRetInstrNotImplemented++;
13948 else
13949 pVCpu->iem.s.cRetErrStatuses++;
13950 }
13951 else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
13952 {
13953 pVCpu->iem.s.cRetPassUpStatus++;
13954 rcStrict = pVCpu->iem.s.rcPassUp;
13955 }
13956
13957 return rcStrict;
13958}
13959
13960
13961/**
13962 * The actual code execution bits of IEMExecOne, IEMExecOneEx, and
13963 * IEMExecOneWithPrefetchedByPC.
13964 *
13965 * Similar code is found in IEMExecLots.
13966 *
13967 * @return Strict VBox status code.
13968 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
13969 * @param fExecuteInhibit If set, execute the instruction following CLI,
13970 * POP SS and MOV SS,GR.
13971 * @param pszFunction The calling function name.
13972 */
13973DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction)
13974{
13975 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
13976 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
13977 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
13978 RT_NOREF_PV(pszFunction);
13979
13980#ifdef IEM_WITH_SETJMP
13981 VBOXSTRICTRC rcStrict;
13982 jmp_buf JmpBuf;
13983 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
13984 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
13985 if ((rcStrict = setjmp(JmpBuf)) == 0)
13986 {
13987 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
13988 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
13989 }
13990 else
13991 pVCpu->iem.s.cLongJumps++;
13992 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
13993#else
13994 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
13995 VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
13996#endif
13997 if (rcStrict == VINF_SUCCESS)
13998 pVCpu->iem.s.cInstructions++;
13999 if (pVCpu->iem.s.cActiveMappings > 0)
14000 {
14001 Assert(rcStrict != VINF_SUCCESS);
14002 iemMemRollback(pVCpu);
14003 }
14004 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
14005 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
14006 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
14007
14008//#ifdef DEBUG
14009// AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr || rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));
14010//#endif
14011
14012#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
14013 /*
14014 * Perform any VMX nested-guest instruction boundary actions.
14015 *
14016 * If any of these causes a VM-exit, we must skip executing the next
14017 * instruction (would run into stale page tables). A VM-exit makes sure
14018 * there is no interrupt-inhibition, so that should ensure we don't go
14019 * to try execute the next instruction. Clearing fExecuteInhibit is
14020 * problematic because of the setjmp/longjmp clobbering above.
14021 */
14022 if ( rcStrict == VINF_SUCCESS
14023 && VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
14024 | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW))
14025 rcStrict = iemHandleNestedInstructionBoundraryFFs(pVCpu, rcStrict);
14026#endif
14027
14028 /* Execute the next instruction as well if a cli, pop ss or
14029 mov ss, Gr has just completed successfully. */
14030 if ( fExecuteInhibit
14031 && rcStrict == VINF_SUCCESS
14032 && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
14033 && EMIsInhibitInterruptsActive(pVCpu))
14034 {
14035 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fBypassHandlers);
14036 if (rcStrict == VINF_SUCCESS)
14037 {
14038#ifdef LOG_ENABLED
14039 iemLogCurInstr(pVCpu, false, pszFunction);
14040#endif
14041#ifdef IEM_WITH_SETJMP
14042 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
14043 if ((rcStrict = setjmp(JmpBuf)) == 0)
14044 {
14045 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
14046 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14047 }
14048 else
14049 pVCpu->iem.s.cLongJumps++;
14050 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
14051#else
14052 IEM_OPCODE_GET_NEXT_U8(&b);
14053 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14054#endif
14055 if (rcStrict == VINF_SUCCESS)
14056 pVCpu->iem.s.cInstructions++;
14057 if (pVCpu->iem.s.cActiveMappings > 0)
14058 {
14059 Assert(rcStrict != VINF_SUCCESS);
14060 iemMemRollback(pVCpu);
14061 }
14062 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
14063 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
14064 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
14065 }
14066 else if (pVCpu->iem.s.cActiveMappings > 0)
14067 iemMemRollback(pVCpu);
14068 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); /* hope this is correct for all exceptional cases... */
14069 }
14070
14071 /*
14072 * Return value fiddling, statistics and sanity assertions.
14073 */
14074 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14075
14076 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
14077 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
14078 return rcStrict;
14079}
14080
14081
14082/**
14083 * Execute one instruction.
14084 *
14085 * @return Strict VBox status code.
14086 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
14087 */
14088VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu)
14089{
14090#ifdef LOG_ENABLED
14091 iemLogCurInstr(pVCpu, true, "IEMExecOne");
14092#endif
14093
14094 /*
14095 * Do the decoding and emulation.
14096 */
14097 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
14098 if (rcStrict == VINF_SUCCESS)
14099 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");
14100 else if (pVCpu->iem.s.cActiveMappings > 0)
14101 iemMemRollback(pVCpu);
14102
14103 if (rcStrict != VINF_SUCCESS)
14104 LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
14105 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
14106 return rcStrict;
14107}
14108
14109
14110VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
14111{
14112 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14113
14114 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
14115 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
14116 if (rcStrict == VINF_SUCCESS)
14117 {
14118 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx");
14119 if (pcbWritten)
14120 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
14121 }
14122 else if (pVCpu->iem.s.cActiveMappings > 0)
14123 iemMemRollback(pVCpu);
14124
14125 return rcStrict;
14126}
14127
14128
14129VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
14130 const void *pvOpcodeBytes, size_t cbOpcodeBytes)
14131{
14132 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14133
14134 VBOXSTRICTRC rcStrict;
14135 if ( cbOpcodeBytes
14136 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
14137 {
14138 iemInitDecoder(pVCpu, false);
14139#ifdef IEM_WITH_CODE_TLB
14140 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
14141 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
14142 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
14143 pVCpu->iem.s.offCurInstrStart = 0;
14144 pVCpu->iem.s.offInstrNextByte = 0;
14145#else
14146 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
14147 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
14148#endif
14149 rcStrict = VINF_SUCCESS;
14150 }
14151 else
14152 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
14153 if (rcStrict == VINF_SUCCESS)
14154 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");
14155 else if (pVCpu->iem.s.cActiveMappings > 0)
14156 iemMemRollback(pVCpu);
14157
14158 return rcStrict;
14159}
14160
14161
14162VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
14163{
14164 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14165
14166 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
14167 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true);
14168 if (rcStrict == VINF_SUCCESS)
14169 {
14170 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx");
14171 if (pcbWritten)
14172 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
14173 }
14174 else if (pVCpu->iem.s.cActiveMappings > 0)
14175 iemMemRollback(pVCpu);
14176
14177 return rcStrict;
14178}
14179
14180
14181VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
14182 const void *pvOpcodeBytes, size_t cbOpcodeBytes)
14183{
14184 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14185
14186 VBOXSTRICTRC rcStrict;
14187 if ( cbOpcodeBytes
14188 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
14189 {
14190 iemInitDecoder(pVCpu, true);
14191#ifdef IEM_WITH_CODE_TLB
14192 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
14193 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
14194 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
14195 pVCpu->iem.s.offCurInstrStart = 0;
14196 pVCpu->iem.s.offInstrNextByte = 0;
14197#else
14198 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
14199 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
14200#endif
14201 rcStrict = VINF_SUCCESS;
14202 }
14203 else
14204 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true);
14205 if (rcStrict == VINF_SUCCESS)
14206 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");
14207 else if (pVCpu->iem.s.cActiveMappings > 0)
14208 iemMemRollback(pVCpu);
14209
14210 return rcStrict;
14211}
14212
14213
14214/**
14215 * For debugging DISGetParamSize, may come in handy.
14216 *
14217 * @returns Strict VBox status code.
14218 * @param pVCpu The cross context virtual CPU structure of the
14219 * calling EMT.
14220 * @param pCtxCore The context core structure.
14221 * @param OpcodeBytesPC The PC of the opcode bytes.
14222 * @param pvOpcodeBytes Prefeched opcode bytes.
14223 * @param cbOpcodeBytes Number of prefetched bytes.
14224 * @param pcbWritten Where to return the number of bytes written.
14225 * Optional.
14226 */
14227VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPCWritten(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
14228 const void *pvOpcodeBytes, size_t cbOpcodeBytes,
14229 uint32_t *pcbWritten)
14230{
14231 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14232
14233 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
14234 VBOXSTRICTRC rcStrict;
14235 if ( cbOpcodeBytes
14236 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
14237 {
14238 iemInitDecoder(pVCpu, true);
14239#ifdef IEM_WITH_CODE_TLB
14240 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
14241 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
14242 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
14243 pVCpu->iem.s.offCurInstrStart = 0;
14244 pVCpu->iem.s.offInstrNextByte = 0;
14245#else
14246 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
14247 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
14248#endif
14249 rcStrict = VINF_SUCCESS;
14250 }
14251 else
14252 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true);
14253 if (rcStrict == VINF_SUCCESS)
14254 {
14255 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPCWritten");
14256 if (pcbWritten)
14257 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
14258 }
14259 else if (pVCpu->iem.s.cActiveMappings > 0)
14260 iemMemRollback(pVCpu);
14261
14262 return rcStrict;
14263}
14264
14265
14266VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions)
14267{
14268 uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;
14269 AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate));
14270
14271 /*
14272 * See if there is an interrupt pending in TRPM, inject it if we can.
14273 */
14274 /** @todo Can we centralize this under CPUMCanInjectInterrupt()? */
14275#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
14276 bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);
14277 if (fIntrEnabled)
14278 {
14279 if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu)))
14280 fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
14281 else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
14282 fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu));
14283 else
14284 {
14285 Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
14286 fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu));
14287 }
14288 }
14289#else
14290 bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
14291#endif
14292
14293 /** @todo What if we are injecting an exception and not an interrupt? Is that
14294 * possible here? For now we assert it is indeed only an interrupt. */
14295 if ( fIntrEnabled
14296 && TRPMHasTrap(pVCpu)
14297 && EMGetInhibitInterruptsPC(pVCpu) != pVCpu->cpum.GstCtx.rip)
14298 {
14299 uint8_t u8TrapNo;
14300 TRPMEVENT enmType;
14301 uint32_t uErrCode;
14302 RTGCPTR uCr2;
14303 int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /* pu8InstLen */, NULL /* fIcebp */);
14304 AssertRC(rc2);
14305 Assert(enmType == TRPM_HARDWARE_INT);
14306 VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /* cbInstr */);
14307 TRPMResetTrap(pVCpu);
14308#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
14309 /* Injecting an event may cause a VM-exit. */
14310 if ( rcStrict != VINF_SUCCESS
14311 && rcStrict != VINF_IEM_RAISED_XCPT)
14312 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
14313#else
14314 NOREF(rcStrict);
14315#endif
14316 }
14317
14318 /*
14319 * Initial decoder init w/ prefetch, then setup setjmp.
14320 */
14321 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
14322 if (rcStrict == VINF_SUCCESS)
14323 {
14324#ifdef IEM_WITH_SETJMP
14325 jmp_buf JmpBuf;
14326 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
14327 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
14328 pVCpu->iem.s.cActiveMappings = 0;
14329 if ((rcStrict = setjmp(JmpBuf)) == 0)
14330#endif
14331 {
14332 /*
14333 * The run loop. We limit ourselves to 4096 instructions right now.
14334 */
14335 uint32_t cMaxInstructionsGccStupidity = cMaxInstructions;
14336 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
14337 for (;;)
14338 {
14339 /*
14340 * Log the state.
14341 */
14342#ifdef LOG_ENABLED
14343 iemLogCurInstr(pVCpu, true, "IEMExecLots");
14344#endif
14345
14346 /*
14347 * Do the decoding and emulation.
14348 */
14349 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
14350 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14351 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14352 {
14353 Assert(pVCpu->iem.s.cActiveMappings == 0);
14354 pVCpu->iem.s.cInstructions++;
14355 if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
14356 {
14357 uint64_t fCpu = pVCpu->fLocalForcedActions
14358 & ( VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
14359 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
14360 | VMCPU_FF_TLB_FLUSH
14361 | VMCPU_FF_INHIBIT_INTERRUPTS
14362 | VMCPU_FF_BLOCK_NMIS
14363 | VMCPU_FF_UNHALT ));
14364
14365 if (RT_LIKELY( ( !fCpu
14366 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
14367 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
14368 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
14369 {
14370 if (cMaxInstructionsGccStupidity-- > 0)
14371 {
14372 /* Poll timers every now an then according to the caller's specs. */
14373 if ( (cMaxInstructionsGccStupidity & cPollRate) != 0
14374 || !TMTimerPollBool(pVM, pVCpu))
14375 {
14376 Assert(pVCpu->iem.s.cActiveMappings == 0);
14377 iemReInitDecoder(pVCpu);
14378 continue;
14379 }
14380 }
14381 }
14382 }
14383 Assert(pVCpu->iem.s.cActiveMappings == 0);
14384 }
14385 else if (pVCpu->iem.s.cActiveMappings > 0)
14386 iemMemRollback(pVCpu);
14387 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14388 break;
14389 }
14390 }
14391#ifdef IEM_WITH_SETJMP
14392 else
14393 {
14394 if (pVCpu->iem.s.cActiveMappings > 0)
14395 iemMemRollback(pVCpu);
14396# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
14397 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14398# endif
14399 pVCpu->iem.s.cLongJumps++;
14400 }
14401 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
14402#endif
14403
14404 /*
14405 * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
14406 */
14407 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
14408 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
14409 }
14410 else
14411 {
14412 if (pVCpu->iem.s.cActiveMappings > 0)
14413 iemMemRollback(pVCpu);
14414
14415#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
14416 /*
14417 * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
14418 * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
14419 */
14420 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14421#endif
14422 }
14423
14424 /*
14425 * Maybe re-enter raw-mode and log.
14426 */
14427 if (rcStrict != VINF_SUCCESS)
14428 LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
14429 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
14430 if (pcInstructions)
14431 *pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;
14432 return rcStrict;
14433}
14434
14435
14436/**
14437 * Interface used by EMExecuteExec, does exit statistics and limits.
14438 *
14439 * @returns Strict VBox status code.
14440 * @param pVCpu The cross context virtual CPU structure.
14441 * @param fWillExit To be defined.
14442 * @param cMinInstructions Minimum number of instructions to execute before checking for FFs.
14443 * @param cMaxInstructions Maximum number of instructions to execute.
14444 * @param cMaxInstructionsWithoutExits
14445 * The max number of instructions without exits.
14446 * @param pStats Where to return statistics.
14447 */
14448VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,
14449 uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)
14450{
14451 NOREF(fWillExit); /** @todo define flexible exit crits */
14452
14453 /*
14454 * Initialize return stats.
14455 */
14456 pStats->cInstructions = 0;
14457 pStats->cExits = 0;
14458 pStats->cMaxExitDistance = 0;
14459 pStats->cReserved = 0;
14460
14461 /*
14462 * Initial decoder init w/ prefetch, then setup setjmp.
14463 */
14464 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
14465 if (rcStrict == VINF_SUCCESS)
14466 {
14467#ifdef IEM_WITH_SETJMP
14468 jmp_buf JmpBuf;
14469 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
14470 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
14471 pVCpu->iem.s.cActiveMappings = 0;
14472 if ((rcStrict = setjmp(JmpBuf)) == 0)
14473#endif
14474 {
14475#ifdef IN_RING0
14476 bool const fCheckPreemptionPending = !RTThreadPreemptIsPossible() || !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
14477#endif
14478 uint32_t cInstructionSinceLastExit = 0;
14479
14480 /*
14481 * The run loop. We limit ourselves to 4096 instructions right now.
14482 */
14483 PVM pVM = pVCpu->CTX_SUFF(pVM);
14484 for (;;)
14485 {
14486 /*
14487 * Log the state.
14488 */
14489#ifdef LOG_ENABLED
14490 iemLogCurInstr(pVCpu, true, "IEMExecForExits");
14491#endif
14492
14493 /*
14494 * Do the decoding and emulation.
14495 */
14496 uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;
14497
14498 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
14499 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14500
14501 if ( cPotentialExits != pVCpu->iem.s.cPotentialExits
14502 && cInstructionSinceLastExit > 0 /* don't count the first */ )
14503 {
14504 pStats->cExits += 1;
14505 if (cInstructionSinceLastExit > pStats->cMaxExitDistance)
14506 pStats->cMaxExitDistance = cInstructionSinceLastExit;
14507 cInstructionSinceLastExit = 0;
14508 }
14509
14510 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14511 {
14512 Assert(pVCpu->iem.s.cActiveMappings == 0);
14513 pVCpu->iem.s.cInstructions++;
14514 pStats->cInstructions++;
14515 cInstructionSinceLastExit++;
14516 if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
14517 {
14518 uint64_t fCpu = pVCpu->fLocalForcedActions
14519 & ( VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
14520 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
14521 | VMCPU_FF_TLB_FLUSH
14522 | VMCPU_FF_INHIBIT_INTERRUPTS
14523 | VMCPU_FF_BLOCK_NMIS
14524 | VMCPU_FF_UNHALT ));
14525
14526 if (RT_LIKELY( ( ( !fCpu
14527 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
14528 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))
14529 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )
14530 || pStats->cInstructions < cMinInstructions))
14531 {
14532 if (pStats->cInstructions < cMaxInstructions)
14533 {
14534 if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)
14535 {
14536#ifdef IN_RING0
14537 if ( !fCheckPreemptionPending
14538 || !RTThreadPreemptIsPending(NIL_RTTHREAD))
14539#endif
14540 {
14541 Assert(pVCpu->iem.s.cActiveMappings == 0);
14542 iemReInitDecoder(pVCpu);
14543 continue;
14544 }
14545#ifdef IN_RING0
14546 rcStrict = VINF_EM_RAW_INTERRUPT;
14547 break;
14548#endif
14549 }
14550 }
14551 }
14552 Assert(!(fCpu & VMCPU_FF_IEM));
14553 }
14554 Assert(pVCpu->iem.s.cActiveMappings == 0);
14555 }
14556 else if (pVCpu->iem.s.cActiveMappings > 0)
14557 iemMemRollback(pVCpu);
14558 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14559 break;
14560 }
14561 }
14562#ifdef IEM_WITH_SETJMP
14563 else
14564 {
14565 if (pVCpu->iem.s.cActiveMappings > 0)
14566 iemMemRollback(pVCpu);
14567 pVCpu->iem.s.cLongJumps++;
14568 }
14569 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
14570#endif
14571
14572 /*
14573 * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
14574 */
14575 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
14576 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
14577 }
14578 else
14579 {
14580 if (pVCpu->iem.s.cActiveMappings > 0)
14581 iemMemRollback(pVCpu);
14582
14583#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
14584 /*
14585 * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
14586 * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
14587 */
14588 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14589#endif
14590 }
14591
14592 /*
14593 * Maybe re-enter raw-mode and log.
14594 */
14595 if (rcStrict != VINF_SUCCESS)
14596 LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",
14597 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,
14598 pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));
14599 return rcStrict;
14600}
14601
14602
14603/**
14604 * Injects a trap, fault, abort, software interrupt or external interrupt.
14605 *
14606 * The parameter list matches TRPMQueryTrapAll pretty closely.
14607 *
14608 * @returns Strict VBox status code.
14609 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
14610 * @param u8TrapNo The trap number.
14611 * @param enmType What type is it (trap/fault/abort), software
14612 * interrupt or hardware interrupt.
14613 * @param uErrCode The error code if applicable.
14614 * @param uCr2 The CR2 value if applicable.
14615 * @param cbInstr The instruction length (only relevant for
14616 * software interrupts).
14617 */
14618VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
14619 uint8_t cbInstr)
14620{
14621 iemInitDecoder(pVCpu, false);
14622#ifdef DBGFTRACE_ENABLED
14623 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
14624 u8TrapNo, enmType, uErrCode, uCr2);
14625#endif
14626
14627 uint32_t fFlags;
14628 switch (enmType)
14629 {
14630 case TRPM_HARDWARE_INT:
14631 Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
14632 fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
14633 uErrCode = uCr2 = 0;
14634 break;
14635
14636 case TRPM_SOFTWARE_INT:
14637 Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
14638 fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
14639 uErrCode = uCr2 = 0;
14640 break;
14641
14642 case TRPM_TRAP:
14643 Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
14644 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
14645 if (u8TrapNo == X86_XCPT_PF)
14646 fFlags |= IEM_XCPT_FLAGS_CR2;
14647 switch (u8TrapNo)
14648 {
14649 case X86_XCPT_DF:
14650 case X86_XCPT_TS:
14651 case X86_XCPT_NP:
14652 case X86_XCPT_SS:
14653 case X86_XCPT_PF:
14654 case X86_XCPT_AC:
14655 fFlags |= IEM_XCPT_FLAGS_ERR;
14656 break;
14657 }
14658 break;
14659
14660 IEM_NOT_REACHED_DEFAULT_CASE_RET();
14661 }
14662
14663 VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
14664
14665 if (pVCpu->iem.s.cActiveMappings > 0)
14666 iemMemRollback(pVCpu);
14667
14668 return rcStrict;
14669}
14670
14671
14672/**
14673 * Injects the active TRPM event.
14674 *
14675 * @returns Strict VBox status code.
14676 * @param pVCpu The cross context virtual CPU structure.
14677 */
14678VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu)
14679{
14680#ifndef IEM_IMPLEMENTS_TASKSWITCH
14681 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
14682#else
14683 uint8_t u8TrapNo;
14684 TRPMEVENT enmType;
14685 uint32_t uErrCode;
14686 RTGCUINTPTR uCr2;
14687 uint8_t cbInstr;
14688 int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */);
14689 if (RT_FAILURE(rc))
14690 return rc;
14691
14692 /** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle
14693 * ICEBP \#DB injection as a special case. */
14694 VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
14695#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
14696 if (rcStrict == VINF_SVM_VMEXIT)
14697 rcStrict = VINF_SUCCESS;
14698#endif
14699#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
14700 if (rcStrict == VINF_VMX_VMEXIT)
14701 rcStrict = VINF_SUCCESS;
14702#endif
14703 /** @todo Are there any other codes that imply the event was successfully
14704 * delivered to the guest? See @bugref{6607}. */
14705 if ( rcStrict == VINF_SUCCESS
14706 || rcStrict == VINF_IEM_RAISED_XCPT)
14707 TRPMResetTrap(pVCpu);
14708
14709 return rcStrict;
14710#endif
14711}
14712
14713
14714VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
14715{
14716 RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
14717 return VERR_NOT_IMPLEMENTED;
14718}
14719
14720
14721VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
14722{
14723 RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
14724 return VERR_NOT_IMPLEMENTED;
14725}
14726
14727
14728#if 0 /* The IRET-to-v8086 mode in PATM is very optimistic, so I don't dare do this yet. */
14729/**
14730 * Executes a IRET instruction with default operand size.
14731 *
14732 * This is for PATM.
14733 *
14734 * @returns VBox status code.
14735 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
14736 * @param pCtxCore The register frame.
14737 */
14738VMM_INT_DECL(int) IEMExecInstr_iret(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore)
14739{
14740 PCPUMCTX pCtx = IEM_GET_CTX(pVCpu);
14741
14742 iemCtxCoreToCtx(pCtx, pCtxCore);
14743 iemInitDecoder(pVCpu);
14744 VBOXSTRICTRC rcStrict = iemCImpl_iret(pVCpu, 1, pVCpu->iem.s.enmDefOpSize);
14745 if (rcStrict == VINF_SUCCESS)
14746 iemCtxToCtxCore(pCtxCore, pCtx);
14747 else
14748 LogFlow(("IEMExecInstr_iret: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
14749 pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
14750 return rcStrict;
14751}
14752#endif
14753
14754
14755/**
14756 * Macro used by the IEMExec* method to check the given instruction length.
14757 *
14758 * Will return on failure!
14759 *
14760 * @param a_cbInstr The given instruction length.
14761 * @param a_cbMin The minimum length.
14762 */
14763#define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
14764 AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
14765 ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
14766
14767
14768/**
14769 * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
14770 *
14771 * Only calling iemRCRawMaybeReenter in raw-mode, obviously.
14772 *
14773 * @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
14774 * @param pVCpu The cross context virtual CPU structure of the calling thread.
14775 * @param rcStrict The status code to fiddle.
14776 */
14777DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
14778{
14779 iemUninitExec(pVCpu);
14780 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
14781}
14782
14783
14784/**
14785 * Interface for HM and EM for executing string I/O OUT (write) instructions.
14786 *
14787 * This API ASSUMES that the caller has already verified that the guest code is
14788 * allowed to access the I/O port. (The I/O port is in the DX register in the
14789 * guest state.)
14790 *
14791 * @returns Strict VBox status code.
14792 * @param pVCpu The cross context virtual CPU structure.
14793 * @param cbValue The size of the I/O port access (1, 2, or 4).
14794 * @param enmAddrMode The addressing mode.
14795 * @param fRepPrefix Indicates whether a repeat prefix is used
14796 * (doesn't matter which for this instruction).
14797 * @param cbInstr The instruction length in bytes.
14798 * @param iEffSeg The effective segment address.
14799 * @param fIoChecked Whether the access to the I/O port has been
14800 * checked or not. It's typically checked in the
14801 * HM scenario.
14802 */
14803VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
14804 bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked)
14805{
14806 AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
14807 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
14808
14809 /*
14810 * State init.
14811 */
14812 iemInitExec(pVCpu, false /*fBypassHandlers*/);
14813
14814 /*
14815 * Switch orgy for getting to the right handler.
14816 */
14817 VBOXSTRICTRC rcStrict;
14818 if (fRepPrefix)
14819 {
14820 switch (enmAddrMode)
14821 {
14822 case IEMMODE_16BIT:
14823 switch (cbValue)
14824 {
14825 case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14826 case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14827 case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14828 default:
14829 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14830 }
14831 break;
14832
14833 case IEMMODE_32BIT:
14834 switch (cbValue)
14835 {
14836 case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14837 case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14838 case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14839 default:
14840 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14841 }
14842 break;
14843
14844 case IEMMODE_64BIT:
14845 switch (cbValue)
14846 {
14847 case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14848 case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14849 case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14850 default:
14851 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14852 }
14853 break;
14854
14855 default:
14856 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
14857 }
14858 }
14859 else
14860 {
14861 switch (enmAddrMode)
14862 {
14863 case IEMMODE_16BIT:
14864 switch (cbValue)
14865 {
14866 case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14867 case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14868 case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14869 default:
14870 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14871 }
14872 break;
14873
14874 case IEMMODE_32BIT:
14875 switch (cbValue)
14876 {
14877 case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14878 case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14879 case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14880 default:
14881 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14882 }
14883 break;
14884
14885 case IEMMODE_64BIT:
14886 switch (cbValue)
14887 {
14888 case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14889 case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14890 case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14891 default:
14892 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14893 }
14894 break;
14895
14896 default:
14897 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
14898 }
14899 }
14900
14901 if (pVCpu->iem.s.cActiveMappings)
14902 iemMemRollback(pVCpu);
14903
14904 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
14905}
14906
14907
14908/**
14909 * Interface for HM and EM for executing string I/O IN (read) instructions.
14910 *
14911 * This API ASSUMES that the caller has already verified that the guest code is
14912 * allowed to access the I/O port. (The I/O port is in the DX register in the
14913 * guest state.)
14914 *
14915 * @returns Strict VBox status code.
14916 * @param pVCpu The cross context virtual CPU structure.
14917 * @param cbValue The size of the I/O port access (1, 2, or 4).
14918 * @param enmAddrMode The addressing mode.
14919 * @param fRepPrefix Indicates whether a repeat prefix is used
14920 * (doesn't matter which for this instruction).
14921 * @param cbInstr The instruction length in bytes.
14922 * @param fIoChecked Whether the access to the I/O port has been
14923 * checked or not. It's typically checked in the
14924 * HM scenario.
14925 */
14926VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
14927 bool fRepPrefix, uint8_t cbInstr, bool fIoChecked)
14928{
14929 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
14930
14931 /*
14932 * State init.
14933 */
14934 iemInitExec(pVCpu, false /*fBypassHandlers*/);
14935
14936 /*
14937 * Switch orgy for getting to the right handler.
14938 */
14939 VBOXSTRICTRC rcStrict;
14940 if (fRepPrefix)
14941 {
14942 switch (enmAddrMode)
14943 {
14944 case IEMMODE_16BIT:
14945 switch (cbValue)
14946 {
14947 case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
14948 case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
14949 case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
14950 default:
14951 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14952 }
14953 break;
14954
14955 case IEMMODE_32BIT:
14956 switch (cbValue)
14957 {
14958 case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
14959 case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
14960 case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
14961 default:
14962 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14963 }
14964 break;
14965
14966 case IEMMODE_64BIT:
14967 switch (cbValue)
14968 {
14969 case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
14970 case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
14971 case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
14972 default:
14973 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14974 }
14975 break;
14976
14977 default:
14978 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
14979 }
14980 }
14981 else
14982 {
14983 switch (enmAddrMode)
14984 {
14985 case IEMMODE_16BIT:
14986 switch (cbValue)
14987 {
14988 case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
14989 case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
14990 case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
14991 default:
14992 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14993 }
14994 break;
14995
14996 case IEMMODE_32BIT:
14997 switch (cbValue)
14998 {
14999 case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
15000 case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
15001 case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
15002 default:
15003 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
15004 }
15005 break;
15006
15007 case IEMMODE_64BIT:
15008 switch (cbValue)
15009 {
15010 case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
15011 case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
15012 case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
15013 default:
15014 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
15015 }
15016 break;
15017
15018 default:
15019 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
15020 }
15021 }
15022
15023 Assert(pVCpu->iem.s.cActiveMappings == 0 || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
15024 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15025}
15026
15027
15028/**
15029 * Interface for rawmode to write execute an OUT instruction.
15030 *
15031 * @returns Strict VBox status code.
15032 * @param pVCpu The cross context virtual CPU structure.
15033 * @param cbInstr The instruction length in bytes.
15034 * @param u16Port The port to read.
15035 * @param fImm Whether the port is specified using an immediate operand or
15036 * using the implicit DX register.
15037 * @param cbReg The register size.
15038 *
15039 * @remarks In ring-0 not all of the state needs to be synced in.
15040 */
15041VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
15042{
15043 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
15044 Assert(cbReg <= 4 && cbReg != 3);
15045
15046 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15047 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_out, u16Port, fImm, cbReg);
15048 Assert(!pVCpu->iem.s.cActiveMappings);
15049 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15050}
15051
15052
15053/**
15054 * Interface for rawmode to write execute an IN instruction.
15055 *
15056 * @returns Strict VBox status code.
15057 * @param pVCpu The cross context virtual CPU structure.
15058 * @param cbInstr The instruction length in bytes.
15059 * @param u16Port The port to read.
15060 * @param fImm Whether the port is specified using an immediate operand or
15061 * using the implicit DX.
15062 * @param cbReg The register size.
15063 */
15064VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
15065{
15066 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
15067 Assert(cbReg <= 4 && cbReg != 3);
15068
15069 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15070 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_in, u16Port, fImm, cbReg);
15071 Assert(!pVCpu->iem.s.cActiveMappings);
15072 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15073}
15074
15075
15076/**
15077 * Interface for HM and EM to write to a CRx register.
15078 *
15079 * @returns Strict VBox status code.
15080 * @param pVCpu The cross context virtual CPU structure.
15081 * @param cbInstr The instruction length in bytes.
15082 * @param iCrReg The control register number (destination).
15083 * @param iGReg The general purpose register number (source).
15084 *
15085 * @remarks In ring-0 not all of the state needs to be synced in.
15086 */
15087VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg)
15088{
15089 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15090 Assert(iCrReg < 16);
15091 Assert(iGReg < 16);
15092
15093 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15094 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg);
15095 Assert(!pVCpu->iem.s.cActiveMappings);
15096 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15097}
15098
15099
15100/**
15101 * Interface for HM and EM to read from a CRx register.
15102 *
15103 * @returns Strict VBox status code.
15104 * @param pVCpu The cross context virtual CPU structure.
15105 * @param cbInstr The instruction length in bytes.
15106 * @param iGReg The general purpose register number (destination).
15107 * @param iCrReg The control register number (source).
15108 *
15109 * @remarks In ring-0 not all of the state needs to be synced in.
15110 */
15111VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15112{
15113 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15114 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4
15115 | CPUMCTX_EXTRN_APIC_TPR);
15116 Assert(iCrReg < 16);
15117 Assert(iGReg < 16);
15118
15119 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15120 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg);
15121 Assert(!pVCpu->iem.s.cActiveMappings);
15122 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15123}
15124
15125
15126/**
15127 * Interface for HM and EM to clear the CR0[TS] bit.
15128 *
15129 * @returns Strict VBox status code.
15130 * @param pVCpu The cross context virtual CPU structure.
15131 * @param cbInstr The instruction length in bytes.
15132 *
15133 * @remarks In ring-0 not all of the state needs to be synced in.
15134 */
15135VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPUCC pVCpu, uint8_t cbInstr)
15136{
15137 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15138
15139 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15140 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts);
15141 Assert(!pVCpu->iem.s.cActiveMappings);
15142 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15143}
15144
15145
15146/**
15147 * Interface for HM and EM to emulate the LMSW instruction (loads CR0).
15148 *
15149 * @returns Strict VBox status code.
15150 * @param pVCpu The cross context virtual CPU structure.
15151 * @param cbInstr The instruction length in bytes.
15152 * @param uValue The value to load into CR0.
15153 * @param GCPtrEffDst The guest-linear address if the LMSW instruction has a
15154 * memory operand. Otherwise pass NIL_RTGCPTR.
15155 *
15156 * @remarks In ring-0 not all of the state needs to be synced in.
15157 */
15158VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uValue, RTGCPTR GCPtrEffDst)
15159{
15160 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15161
15162 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15163 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_lmsw, uValue, GCPtrEffDst);
15164 Assert(!pVCpu->iem.s.cActiveMappings);
15165 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15166}
15167
15168
15169/**
15170 * Interface for HM and EM to emulate the XSETBV instruction (loads XCRx).
15171 *
15172 * Takes input values in ecx and edx:eax of the CPU context of the calling EMT.
15173 *
15174 * @returns Strict VBox status code.
15175 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15176 * @param cbInstr The instruction length in bytes.
15177 * @remarks In ring-0 not all of the state needs to be synced in.
15178 * @thread EMT(pVCpu)
15179 */
15180VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPUCC pVCpu, uint8_t cbInstr)
15181{
15182 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15183
15184 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15185 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv);
15186 Assert(!pVCpu->iem.s.cActiveMappings);
15187 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15188}
15189
15190
15191/**
15192 * Interface for HM and EM to emulate the WBINVD instruction.
15193 *
15194 * @returns Strict VBox status code.
15195 * @param pVCpu The cross context virtual CPU structure.
15196 * @param cbInstr The instruction length in bytes.
15197 *
15198 * @remarks In ring-0 not all of the state needs to be synced in.
15199 */
15200VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPUCC pVCpu, uint8_t cbInstr)
15201{
15202 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15203
15204 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15205 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd);
15206 Assert(!pVCpu->iem.s.cActiveMappings);
15207 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15208}
15209
15210
15211/**
15212 * Interface for HM and EM to emulate the INVD instruction.
15213 *
15214 * @returns Strict VBox status code.
15215 * @param pVCpu The cross context virtual CPU structure.
15216 * @param cbInstr The instruction length in bytes.
15217 *
15218 * @remarks In ring-0 not all of the state needs to be synced in.
15219 */
15220VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPUCC pVCpu, uint8_t cbInstr)
15221{
15222 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15223
15224 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15225 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd);
15226 Assert(!pVCpu->iem.s.cActiveMappings);
15227 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15228}
15229
15230
15231/**
15232 * Interface for HM and EM to emulate the INVLPG instruction.
15233 *
15234 * @returns Strict VBox status code.
15235 * @retval VINF_PGM_SYNC_CR3
15236 *
15237 * @param pVCpu The cross context virtual CPU structure.
15238 * @param cbInstr The instruction length in bytes.
15239 * @param GCPtrPage The effective address of the page to invalidate.
15240 *
15241 * @remarks In ring-0 not all of the state needs to be synced in.
15242 */
15243VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage)
15244{
15245 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15246
15247 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15248 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage);
15249 Assert(!pVCpu->iem.s.cActiveMappings);
15250 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15251}
15252
15253
15254/**
15255 * Interface for HM and EM to emulate the INVPCID instruction.
15256 *
15257 * @returns Strict VBox status code.
15258 * @retval VINF_PGM_SYNC_CR3
15259 *
15260 * @param pVCpu The cross context virtual CPU structure.
15261 * @param cbInstr The instruction length in bytes.
15262 * @param iEffSeg The effective segment register.
15263 * @param GCPtrDesc The effective address of the INVPCID descriptor.
15264 * @param uType The invalidation type.
15265 *
15266 * @remarks In ring-0 not all of the state needs to be synced in.
15267 */
15268VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvpcid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDesc,
15269 uint64_t uType)
15270{
15271 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 4);
15272
15273 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15274 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_invpcid, iEffSeg, GCPtrDesc, uType);
15275 Assert(!pVCpu->iem.s.cActiveMappings);
15276 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15277}
15278
15279
15280/**
15281 * Interface for HM and EM to emulate the CPUID instruction.
15282 *
15283 * @returns Strict VBox status code.
15284 *
15285 * @param pVCpu The cross context virtual CPU structure.
15286 * @param cbInstr The instruction length in bytes.
15287 *
15288 * @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX.
15289 */
15290VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPUCC pVCpu, uint8_t cbInstr)
15291{
15292 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15293 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
15294
15295 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15296 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid);
15297 Assert(!pVCpu->iem.s.cActiveMappings);
15298 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15299}
15300
15301
15302/**
15303 * Interface for HM and EM to emulate the RDPMC instruction.
15304 *
15305 * @returns Strict VBox status code.
15306 *
15307 * @param pVCpu The cross context virtual CPU structure.
15308 * @param cbInstr The instruction length in bytes.
15309 *
15310 * @remarks Not all of the state needs to be synced in.
15311 */
15312VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPUCC pVCpu, uint8_t cbInstr)
15313{
15314 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15315 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
15316
15317 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15318 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc);
15319 Assert(!pVCpu->iem.s.cActiveMappings);
15320 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15321}
15322
15323
15324/**
15325 * Interface for HM and EM to emulate the RDTSC instruction.
15326 *
15327 * @returns Strict VBox status code.
15328 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15329 *
15330 * @param pVCpu The cross context virtual CPU structure.
15331 * @param cbInstr The instruction length in bytes.
15332 *
15333 * @remarks Not all of the state needs to be synced in.
15334 */
15335VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPUCC pVCpu, uint8_t cbInstr)
15336{
15337 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15338 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
15339
15340 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15341 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc);
15342 Assert(!pVCpu->iem.s.cActiveMappings);
15343 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15344}
15345
15346
15347/**
15348 * Interface for HM and EM to emulate the RDTSCP instruction.
15349 *
15350 * @returns Strict VBox status code.
15351 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15352 *
15353 * @param pVCpu The cross context virtual CPU structure.
15354 * @param cbInstr The instruction length in bytes.
15355 *
15356 * @remarks Not all of the state needs to be synced in. Recommended
15357 * to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call.
15358 */
15359VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPUCC pVCpu, uint8_t cbInstr)
15360{
15361 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15362 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_TSC_AUX);
15363
15364 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15365 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp);
15366 Assert(!pVCpu->iem.s.cActiveMappings);
15367 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15368}
15369
15370
15371/**
15372 * Interface for HM and EM to emulate the RDMSR instruction.
15373 *
15374 * @returns Strict VBox status code.
15375 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15376 *
15377 * @param pVCpu The cross context virtual CPU structure.
15378 * @param cbInstr The instruction length in bytes.
15379 *
15380 * @remarks Not all of the state needs to be synced in. Requires RCX and
15381 * (currently) all MSRs.
15382 */
15383VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
15384{
15385 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15386 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_ALL_MSRS);
15387
15388 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15389 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr);
15390 Assert(!pVCpu->iem.s.cActiveMappings);
15391 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15392}
15393
15394
15395/**
15396 * Interface for HM and EM to emulate the WRMSR instruction.
15397 *
15398 * @returns Strict VBox status code.
15399 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15400 *
15401 * @param pVCpu The cross context virtual CPU structure.
15402 * @param cbInstr The instruction length in bytes.
15403 *
15404 * @remarks Not all of the state needs to be synced in. Requires RCX, RAX, RDX,
15405 * and (currently) all MSRs.
15406 */
15407VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
15408{
15409 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15410 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
15411 | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_ALL_MSRS);
15412
15413 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15414 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr);
15415 Assert(!pVCpu->iem.s.cActiveMappings);
15416 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15417}
15418
15419
15420/**
15421 * Interface for HM and EM to emulate the MONITOR instruction.
15422 *
15423 * @returns Strict VBox status code.
15424 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15425 *
15426 * @param pVCpu The cross context virtual CPU structure.
15427 * @param cbInstr The instruction length in bytes.
15428 *
15429 * @remarks Not all of the state needs to be synced in.
15430 * @remarks ASSUMES the default segment of DS and no segment override prefixes
15431 * are used.
15432 */
15433VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPUCC pVCpu, uint8_t cbInstr)
15434{
15435 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15436 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
15437
15438 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15439 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS);
15440 Assert(!pVCpu->iem.s.cActiveMappings);
15441 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15442}
15443
15444
15445/**
15446 * Interface for HM and EM to emulate the MWAIT instruction.
15447 *
15448 * @returns Strict VBox status code.
15449 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15450 *
15451 * @param pVCpu The cross context virtual CPU structure.
15452 * @param cbInstr The instruction length in bytes.
15453 *
15454 * @remarks Not all of the state needs to be synced in.
15455 */
15456VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPUCC pVCpu, uint8_t cbInstr)
15457{
15458 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15459 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX);
15460
15461 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15462 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait);
15463 Assert(!pVCpu->iem.s.cActiveMappings);
15464 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15465}
15466
15467
15468/**
15469 * Interface for HM and EM to emulate the HLT instruction.
15470 *
15471 * @returns Strict VBox status code.
15472 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15473 *
15474 * @param pVCpu The cross context virtual CPU structure.
15475 * @param cbInstr The instruction length in bytes.
15476 *
15477 * @remarks Not all of the state needs to be synced in.
15478 */
15479VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPUCC pVCpu, uint8_t cbInstr)
15480{
15481 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
15482
15483 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15484 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt);
15485 Assert(!pVCpu->iem.s.cActiveMappings);
15486 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15487}
15488
15489
15490/**
15491 * Checks if IEM is in the process of delivering an event (interrupt or
15492 * exception).
15493 *
15494 * @returns true if we're in the process of raising an interrupt or exception,
15495 * false otherwise.
15496 * @param pVCpu The cross context virtual CPU structure.
15497 * @param puVector Where to store the vector associated with the
15498 * currently delivered event, optional.
15499 * @param pfFlags Where to store th event delivery flags (see
15500 * IEM_XCPT_FLAGS_XXX), optional.
15501 * @param puErr Where to store the error code associated with the
15502 * event, optional.
15503 * @param puCr2 Where to store the CR2 associated with the event,
15504 * optional.
15505 * @remarks The caller should check the flags to determine if the error code and
15506 * CR2 are valid for the event.
15507 */
15508VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPUCC pVCpu, uint8_t *puVector, uint32_t *pfFlags, uint32_t *puErr, uint64_t *puCr2)
15509{
15510 bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0;
15511 if (fRaisingXcpt)
15512 {
15513 if (puVector)
15514 *puVector = pVCpu->iem.s.uCurXcpt;
15515 if (pfFlags)
15516 *pfFlags = pVCpu->iem.s.fCurXcpt;
15517 if (puErr)
15518 *puErr = pVCpu->iem.s.uCurXcptErr;
15519 if (puCr2)
15520 *puCr2 = pVCpu->iem.s.uCurXcptCr2;
15521 }
15522 return fRaisingXcpt;
15523}
15524
15525#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
15526
15527/**
15528 * Interface for HM and EM to emulate the CLGI instruction.
15529 *
15530 * @returns Strict VBox status code.
15531 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15532 * @param cbInstr The instruction length in bytes.
15533 * @thread EMT(pVCpu)
15534 */
15535VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClgi(PVMCPUCC pVCpu, uint8_t cbInstr)
15536{
15537 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15538
15539 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15540 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clgi);
15541 Assert(!pVCpu->iem.s.cActiveMappings);
15542 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15543}
15544
15545
15546/**
15547 * Interface for HM and EM to emulate the STGI instruction.
15548 *
15549 * @returns Strict VBox status code.
15550 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15551 * @param cbInstr The instruction length in bytes.
15552 * @thread EMT(pVCpu)
15553 */
15554VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedStgi(PVMCPUCC pVCpu, uint8_t cbInstr)
15555{
15556 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15557
15558 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15559 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_stgi);
15560 Assert(!pVCpu->iem.s.cActiveMappings);
15561 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15562}
15563
15564
15565/**
15566 * Interface for HM and EM to emulate the VMLOAD instruction.
15567 *
15568 * @returns Strict VBox status code.
15569 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15570 * @param cbInstr The instruction length in bytes.
15571 * @thread EMT(pVCpu)
15572 */
15573VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmload(PVMCPUCC pVCpu, uint8_t cbInstr)
15574{
15575 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15576
15577 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15578 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmload);
15579 Assert(!pVCpu->iem.s.cActiveMappings);
15580 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15581}
15582
15583
15584/**
15585 * Interface for HM and EM to emulate the VMSAVE instruction.
15586 *
15587 * @returns Strict VBox status code.
15588 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15589 * @param cbInstr The instruction length in bytes.
15590 * @thread EMT(pVCpu)
15591 */
15592VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmsave(PVMCPUCC pVCpu, uint8_t cbInstr)
15593{
15594 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15595
15596 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15597 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmsave);
15598 Assert(!pVCpu->iem.s.cActiveMappings);
15599 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15600}
15601
15602
15603/**
15604 * Interface for HM and EM to emulate the INVLPGA instruction.
15605 *
15606 * @returns Strict VBox status code.
15607 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15608 * @param cbInstr The instruction length in bytes.
15609 * @thread EMT(pVCpu)
15610 */
15611VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpga(PVMCPUCC pVCpu, uint8_t cbInstr)
15612{
15613 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15614
15615 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15616 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invlpga);
15617 Assert(!pVCpu->iem.s.cActiveMappings);
15618 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15619}
15620
15621
15622/**
15623 * Interface for HM and EM to emulate the VMRUN instruction.
15624 *
15625 * @returns Strict VBox status code.
15626 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15627 * @param cbInstr The instruction length in bytes.
15628 * @thread EMT(pVCpu)
15629 */
15630VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmrun(PVMCPUCC pVCpu, uint8_t cbInstr)
15631{
15632 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15633 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMRUN_MASK);
15634
15635 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15636 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmrun);
15637 Assert(!pVCpu->iem.s.cActiveMappings);
15638 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15639}
15640
15641
15642/**
15643 * Interface for HM and EM to emulate \#VMEXIT.
15644 *
15645 * @returns Strict VBox status code.
15646 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15647 * @param uExitCode The exit code.
15648 * @param uExitInfo1 The exit info. 1 field.
15649 * @param uExitInfo2 The exit info. 2 field.
15650 * @thread EMT(pVCpu)
15651 */
15652VMM_INT_DECL(VBOXSTRICTRC) IEMExecSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2)
15653{
15654 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK);
15655 VBOXSTRICTRC rcStrict = iemSvmVmexit(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
15656 if (pVCpu->iem.s.cActiveMappings)
15657 iemMemRollback(pVCpu);
15658 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15659}
15660
15661#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
15662
15663#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15664
15665/**
15666 * Interface for HM and EM to read a VMCS field from the nested-guest VMCS.
15667 *
15668 * It is ASSUMED the caller knows what they're doing. No VMREAD instruction checks
15669 * are performed. Bounds checks are strict builds only.
15670 *
15671 * @param pVmcs Pointer to the virtual VMCS.
15672 * @param u64VmcsField The VMCS field.
15673 * @param pu64Dst Where to store the VMCS value.
15674 *
15675 * @remarks May be called with interrupts disabled.
15676 * @todo This should probably be moved to CPUM someday.
15677 */
15678VMM_INT_DECL(void) IEMReadVmxVmcsField(PCVMXVVMCS pVmcs, uint64_t u64VmcsField, uint64_t *pu64Dst)
15679{
15680 AssertPtr(pVmcs);
15681 AssertPtr(pu64Dst);
15682 iemVmxVmreadNoCheck(pVmcs, pu64Dst, u64VmcsField);
15683}
15684
15685
15686/**
15687 * Interface for HM and EM to write a VMCS field in the nested-guest VMCS.
15688 *
15689 * It is ASSUMED the caller knows what they're doing. No VMWRITE instruction checks
15690 * are performed. Bounds checks are strict builds only.
15691 *
15692 * @param pVmcs Pointer to the virtual VMCS.
15693 * @param u64VmcsField The VMCS field.
15694 * @param u64Val The value to write.
15695 *
15696 * @remarks May be called with interrupts disabled.
15697 * @todo This should probably be moved to CPUM someday.
15698 */
15699VMM_INT_DECL(void) IEMWriteVmxVmcsField(PVMXVVMCS pVmcs, uint64_t u64VmcsField, uint64_t u64Val)
15700{
15701 AssertPtr(pVmcs);
15702 iemVmxVmwriteNoCheck(pVmcs, u64Val, u64VmcsField);
15703}
15704
15705
15706/**
15707 * Interface for HM and EM to virtualize x2APIC MSR accesses.
15708 *
15709 * @returns Strict VBox status code.
15710 * @retval VINF_VMX_MODIFIES_BEHAVIOR if the MSR access was virtualized.
15711 * @retval VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR access must be handled by
15712 * the x2APIC device.
15713 * @retval VERR_OUT_RANGE if the caller must raise \#GP(0).
15714 *
15715 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15716 * @param idMsr The MSR being read.
15717 * @param pu64Value Pointer to the value being written or where to store the
15718 * value being read.
15719 * @param fWrite Whether this is an MSR write or read access.
15720 * @thread EMT(pVCpu)
15721 */
15722VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVirtApicAccessMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value, bool fWrite)
15723{
15724 Assert(pu64Value);
15725
15726 VBOXSTRICTRC rcStrict;
15727 if (fWrite)
15728 rcStrict = iemVmxVirtApicAccessMsrWrite(pVCpu, idMsr, *pu64Value);
15729 else
15730 rcStrict = iemVmxVirtApicAccessMsrRead(pVCpu, idMsr, pu64Value);
15731 Assert(!pVCpu->iem.s.cActiveMappings);
15732 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15733
15734}
15735
15736
15737/**
15738 * Interface for HM and EM to virtualize memory-mapped APIC accesses.
15739 *
15740 * @returns Strict VBox status code.
15741 * @retval VINF_VMX_MODIFIES_BEHAVIOR if the memory access was virtualized.
15742 * @retval VINF_VMX_VMEXIT if the access causes a VM-exit.
15743 *
15744 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15745 * @param pExitInfo Pointer to the VM-exit information.
15746 * @param pExitEventInfo Pointer to the VM-exit event information.
15747 * @thread EMT(pVCpu)
15748 */
15749VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitApicAccess(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
15750{
15751 Assert(pExitInfo);
15752 Assert(pExitEventInfo);
15753 VBOXSTRICTRC rcStrict = iemVmxVmexitApicAccessWithInfo(pVCpu, pExitInfo, pExitEventInfo);
15754 Assert(!pVCpu->iem.s.cActiveMappings);
15755 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15756
15757}
15758
15759
15760/**
15761 * Interface for HM and EM to perform an APIC-write emulation which may cause a
15762 * VM-exit.
15763 *
15764 * @returns Strict VBox status code.
15765 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15766 * @thread EMT(pVCpu)
15767 */
15768VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitApicWrite(PVMCPUCC pVCpu)
15769{
15770 VBOXSTRICTRC rcStrict = iemVmxApicWriteEmulation(pVCpu);
15771 Assert(!pVCpu->iem.s.cActiveMappings);
15772 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15773}
15774
15775
15776/**
15777 * Interface for HM and EM to emulate VM-exit due to expiry of the preemption timer.
15778 *
15779 * @returns Strict VBox status code.
15780 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15781 * @thread EMT(pVCpu)
15782 */
15783VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitPreemptTimer(PVMCPUCC pVCpu)
15784{
15785 VBOXSTRICTRC rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
15786 Assert(!pVCpu->iem.s.cActiveMappings);
15787 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15788}
15789
15790
15791/**
15792 * Interface for HM and EM to emulate VM-exit due to external interrupts.
15793 *
15794 * @returns Strict VBox status code.
15795 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15796 * @param uVector The external interrupt vector (pass 0 if the external
15797 * interrupt is still pending).
15798 * @param fIntPending Whether the external interrupt is pending or
15799 * acknowdledged in the interrupt controller.
15800 * @thread EMT(pVCpu)
15801 */
15802VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitExtInt(PVMCPUCC pVCpu, uint8_t uVector, bool fIntPending)
15803{
15804 VBOXSTRICTRC rcStrict = iemVmxVmexitExtInt(pVCpu, uVector, fIntPending);
15805 Assert(!pVCpu->iem.s.cActiveMappings);
15806 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15807}
15808
15809
15810/**
15811 * Interface for HM and EM to emulate VM-exit due to exceptions.
15812 *
15813 * Exception includes NMIs, software exceptions (those generated by INT3 or
15814 * INTO) and privileged software exceptions (those generated by INT1/ICEBP).
15815 *
15816 * @returns Strict VBox status code.
15817 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15818 * @param pExitInfo Pointer to the VM-exit information.
15819 * @param pExitEventInfo Pointer to the VM-exit event information.
15820 * @thread EMT(pVCpu)
15821 */
15822VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitXcpt(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
15823{
15824 Assert(pExitInfo);
15825 Assert(pExitEventInfo);
15826 VBOXSTRICTRC rcStrict = iemVmxVmexitEventWithInfo(pVCpu, pExitInfo, pExitEventInfo);
15827 Assert(!pVCpu->iem.s.cActiveMappings);
15828 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15829}
15830
15831
15832/**
15833 * Interface for HM and EM to emulate VM-exit due to NMIs.
15834 *
15835 * @returns Strict VBox status code.
15836 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15837 * @thread EMT(pVCpu)
15838 */
15839VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitXcptNmi(PVMCPUCC pVCpu)
15840{
15841 VMXVEXITINFO ExitInfo;
15842 RT_ZERO(ExitInfo);
15843 ExitInfo.uReason = VMX_EXIT_XCPT_OR_NMI;
15844
15845 VMXVEXITEVENTINFO ExitEventInfo;
15846 RT_ZERO(ExitEventInfo);
15847 ExitEventInfo.uExitIntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID, 1)
15848 | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_NMI)
15849 | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, X86_XCPT_NMI);
15850
15851 VBOXSTRICTRC rcStrict = iemVmxVmexitEventWithInfo(pVCpu, &ExitInfo, &ExitEventInfo);
15852 Assert(!pVCpu->iem.s.cActiveMappings);
15853 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15854}
15855
15856
15857/**
15858 * Interface for HM and EM to emulate VM-exit due to a triple-fault.
15859 *
15860 * @returns Strict VBox status code.
15861 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15862 * @thread EMT(pVCpu)
15863 */
15864VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTripleFault(PVMCPUCC pVCpu)
15865{
15866 VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
15867 Assert(!pVCpu->iem.s.cActiveMappings);
15868 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15869}
15870
15871
15872/**
15873 * Interface for HM and EM to emulate VM-exit due to startup-IPI (SIPI).
15874 *
15875 * @returns Strict VBox status code.
15876 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15877 * @param uVector The SIPI vector.
15878 * @thread EMT(pVCpu)
15879 */
15880VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitStartupIpi(PVMCPUCC pVCpu, uint8_t uVector)
15881{
15882 VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_SIPI, uVector);
15883 Assert(!pVCpu->iem.s.cActiveMappings);
15884 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15885}
15886
15887
15888/**
15889 * Interface for HM and EM to emulate a VM-exit.
15890 *
15891 * If a specialized version of a VM-exit handler exists, that must be used instead.
15892 *
15893 * @returns Strict VBox status code.
15894 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15895 * @param uExitReason The VM-exit reason.
15896 * @param u64ExitQual The Exit qualification.
15897 * @thread EMT(pVCpu)
15898 */
15899VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual)
15900{
15901 VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, uExitReason, u64ExitQual);
15902 Assert(!pVCpu->iem.s.cActiveMappings);
15903 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15904}
15905
15906
15907/**
15908 * Interface for HM and EM to emulate a VM-exit due to an instruction.
15909 *
15910 * This is meant to be used for those instructions that VMX provides additional
15911 * decoding information beyond just the instruction length!
15912 *
15913 * @returns Strict VBox status code.
15914 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15915 * @param pExitInfo Pointer to the VM-exit information.
15916 * @thread EMT(pVCpu)
15917 */
15918VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitInstrWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
15919{
15920 VBOXSTRICTRC rcStrict = iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
15921 Assert(!pVCpu->iem.s.cActiveMappings);
15922 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15923}
15924
15925
15926/**
15927 * Interface for HM and EM to emulate a VM-exit due to an instruction.
15928 *
15929 * This is meant to be used for those instructions that VMX provides only the
15930 * instruction length.
15931 *
15932 * @returns Strict VBox status code.
15933 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15934 * @param pExitInfo Pointer to the VM-exit information.
15935 * @param cbInstr The instruction length in bytes.
15936 * @thread EMT(pVCpu)
15937 */
15938VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr)
15939{
15940 VBOXSTRICTRC rcStrict = iemVmxVmexitInstr(pVCpu, uExitReason, cbInstr);
15941 Assert(!pVCpu->iem.s.cActiveMappings);
15942 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15943}
15944
15945
15946/**
15947 * Interface for HM and EM to emulate a trap-like VM-exit (MTF, APIC-write,
15948 * Virtualized-EOI, TPR-below threshold).
15949 *
15950 * @returns Strict VBox status code.
15951 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15952 * @param pExitInfo Pointer to the VM-exit information.
15953 * @thread EMT(pVCpu)
15954 */
15955VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTrapLike(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
15956{
15957 Assert(pExitInfo);
15958 VBOXSTRICTRC rcStrict = iemVmxVmexitTrapLikeWithInfo(pVCpu, pExitInfo);
15959 Assert(!pVCpu->iem.s.cActiveMappings);
15960 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15961}
15962
15963
15964/**
15965 * Interface for HM and EM to emulate a VM-exit due to a task switch.
15966 *
15967 * @returns Strict VBox status code.
15968 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15969 * @param pExitInfo Pointer to the VM-exit information.
15970 * @param pExitEventInfo Pointer to the VM-exit event information.
15971 * @thread EMT(pVCpu)
15972 */
15973VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTaskSwitch(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
15974{
15975 Assert(pExitInfo);
15976 Assert(pExitEventInfo);
15977 Assert(pExitInfo->uReason == VMX_EXIT_TASK_SWITCH);
15978 VBOXSTRICTRC rcStrict = iemVmxVmexitTaskSwitchWithInfo(pVCpu, pExitInfo, pExitEventInfo);
15979 Assert(!pVCpu->iem.s.cActiveMappings);
15980 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15981}
15982
15983
15984/**
15985 * Interface for HM and EM to emulate the VMREAD instruction.
15986 *
15987 * @returns Strict VBox status code.
15988 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15989 * @param pExitInfo Pointer to the VM-exit information.
15990 * @thread EMT(pVCpu)
15991 */
15992VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmread(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
15993{
15994 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
15995 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
15996 Assert(pExitInfo);
15997
15998 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15999
16000 VBOXSTRICTRC rcStrict;
16001 uint8_t const cbInstr = pExitInfo->cbInstr;
16002 bool const fIs64BitMode = RT_BOOL(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
16003 uint64_t const u64FieldEnc = fIs64BitMode
16004 ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2)
16005 : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2);
16006 if (pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand)
16007 {
16008 if (fIs64BitMode)
16009 {
16010 uint64_t *pu64Dst = iemGRegRefU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
16011 rcStrict = iemVmxVmreadReg64(pVCpu, cbInstr, pu64Dst, u64FieldEnc, pExitInfo);
16012 }
16013 else
16014 {
16015 uint32_t *pu32Dst = iemGRegRefU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
16016 rcStrict = iemVmxVmreadReg32(pVCpu, cbInstr, pu32Dst, u64FieldEnc, pExitInfo);
16017 }
16018 }
16019 else
16020 {
16021 RTGCPTR const GCPtrDst = pExitInfo->GCPtrEffAddr;
16022 uint8_t const iEffSeg = pExitInfo->InstrInfo.VmreadVmwrite.iSegReg;
16023 rcStrict = iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u64FieldEnc, pExitInfo);
16024 }
16025 Assert(!pVCpu->iem.s.cActiveMappings);
16026 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16027}
16028
16029
16030/**
16031 * Interface for HM and EM to emulate the VMWRITE instruction.
16032 *
16033 * @returns Strict VBox status code.
16034 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16035 * @param pExitInfo Pointer to the VM-exit information.
16036 * @thread EMT(pVCpu)
16037 */
16038VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmwrite(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16039{
16040 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16041 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
16042 Assert(pExitInfo);
16043
16044 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16045
16046 uint64_t u64Val;
16047 uint8_t iEffSeg;
16048 if (pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand)
16049 {
16050 u64Val = iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
16051 iEffSeg = UINT8_MAX;
16052 }
16053 else
16054 {
16055 u64Val = pExitInfo->GCPtrEffAddr;
16056 iEffSeg = pExitInfo->InstrInfo.VmreadVmwrite.iSegReg;
16057 }
16058 uint8_t const cbInstr = pExitInfo->cbInstr;
16059 uint64_t const u64FieldEnc = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
16060 ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2)
16061 : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2);
16062 VBOXSTRICTRC rcStrict = iemVmxVmwrite(pVCpu, cbInstr, iEffSeg, u64Val, u64FieldEnc, pExitInfo);
16063 Assert(!pVCpu->iem.s.cActiveMappings);
16064 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16065}
16066
16067
16068/**
16069 * Interface for HM and EM to emulate the VMPTRLD instruction.
16070 *
16071 * @returns Strict VBox status code.
16072 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16073 * @param pExitInfo Pointer to the VM-exit information.
16074 * @thread EMT(pVCpu)
16075 */
16076VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmptrld(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16077{
16078 Assert(pExitInfo);
16079 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16080 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
16081
16082 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16083
16084 uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg;
16085 uint8_t const cbInstr = pExitInfo->cbInstr;
16086 RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
16087 VBOXSTRICTRC rcStrict = iemVmxVmptrld(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
16088 Assert(!pVCpu->iem.s.cActiveMappings);
16089 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16090}
16091
16092
16093/**
16094 * Interface for HM and EM to emulate the VMPTRST instruction.
16095 *
16096 * @returns Strict VBox status code.
16097 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16098 * @param pExitInfo Pointer to the VM-exit information.
16099 * @thread EMT(pVCpu)
16100 */
16101VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmptrst(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16102{
16103 Assert(pExitInfo);
16104 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16105 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
16106
16107 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16108
16109 uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg;
16110 uint8_t const cbInstr = pExitInfo->cbInstr;
16111 RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
16112 VBOXSTRICTRC rcStrict = iemVmxVmptrst(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
16113 Assert(!pVCpu->iem.s.cActiveMappings);
16114 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16115}
16116
16117
16118/**
16119 * Interface for HM and EM to emulate the VMCLEAR instruction.
16120 *
16121 * @returns Strict VBox status code.
16122 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16123 * @param pExitInfo Pointer to the VM-exit information.
16124 * @thread EMT(pVCpu)
16125 */
16126VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmclear(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16127{
16128 Assert(pExitInfo);
16129 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16130 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
16131
16132 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16133
16134 uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg;
16135 uint8_t const cbInstr = pExitInfo->cbInstr;
16136 RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
16137 VBOXSTRICTRC rcStrict = iemVmxVmclear(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
16138 Assert(!pVCpu->iem.s.cActiveMappings);
16139 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16140}
16141
16142
16143/**
16144 * Interface for HM and EM to emulate the VMLAUNCH/VMRESUME instruction.
16145 *
16146 * @returns Strict VBox status code.
16147 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16148 * @param cbInstr The instruction length in bytes.
16149 * @param uInstrId The instruction ID (VMXINSTRID_VMLAUNCH or
16150 * VMXINSTRID_VMRESUME).
16151 * @thread EMT(pVCpu)
16152 */
16153VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmlaunchVmresume(PVMCPUCC pVCpu, uint8_t cbInstr, VMXINSTRID uInstrId)
16154{
16155 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
16156 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK);
16157
16158 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16159 VBOXSTRICTRC rcStrict = iemVmxVmlaunchVmresume(pVCpu, cbInstr, uInstrId);
16160 Assert(!pVCpu->iem.s.cActiveMappings);
16161 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16162}
16163
16164
16165/**
16166 * Interface for HM and EM to emulate the VMXON instruction.
16167 *
16168 * @returns Strict VBox status code.
16169 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16170 * @param pExitInfo Pointer to the VM-exit information.
16171 * @thread EMT(pVCpu)
16172 */
16173VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxon(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16174{
16175 Assert(pExitInfo);
16176 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16177 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
16178
16179 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16180
16181 uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg;
16182 uint8_t const cbInstr = pExitInfo->cbInstr;
16183 RTGCPTR const GCPtrVmxon = pExitInfo->GCPtrEffAddr;
16184 VBOXSTRICTRC rcStrict = iemVmxVmxon(pVCpu, cbInstr, iEffSeg, GCPtrVmxon, pExitInfo);
16185 Assert(!pVCpu->iem.s.cActiveMappings);
16186 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16187}
16188
16189
16190/**
16191 * Interface for HM and EM to emulate the VMXOFF instruction.
16192 *
16193 * @returns Strict VBox status code.
16194 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16195 * @param cbInstr The instruction length in bytes.
16196 * @thread EMT(pVCpu)
16197 */
16198VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxoff(PVMCPUCC pVCpu, uint8_t cbInstr)
16199{
16200 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
16201 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
16202
16203 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16204 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmxoff);
16205 Assert(!pVCpu->iem.s.cActiveMappings);
16206 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16207}
16208
16209
16210/**
16211 * Interface for HM and EM to emulate the INVVPID instruction.
16212 *
16213 * @returns Strict VBox status code.
16214 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16215 * @param pExitInfo Pointer to the VM-exit information.
16216 * @thread EMT(pVCpu)
16217 */
16218VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvvpid(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16219{
16220 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 4);
16221 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_HM_VMX_MASK);
16222 Assert(pExitInfo);
16223
16224 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16225
16226 uint8_t const iEffSeg = pExitInfo->InstrInfo.Inv.iSegReg;
16227 uint8_t const cbInstr = pExitInfo->cbInstr;
16228 RTGCPTR const GCPtrInvvpidDesc = pExitInfo->GCPtrEffAddr;
16229 uint64_t const u64InvvpidType = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
16230 ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.Inv.iReg2)
16231 : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.Inv.iReg2);
16232 VBOXSTRICTRC rcStrict = iemVmxInvvpid(pVCpu, cbInstr, iEffSeg, GCPtrInvvpidDesc, u64InvvpidType, pExitInfo);
16233 Assert(!pVCpu->iem.s.cActiveMappings);
16234 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16235}
16236
16237
16238/**
16239 * @callback_method_impl{FNPGMPHYSHANDLER, VMX APIC-access page accesses}
16240 *
16241 * @remarks The @a pvUser argument is currently unused.
16242 */
16243PGM_ALL_CB2_DECL(VBOXSTRICTRC) iemVmxApicAccessPageHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys,
16244 void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType,
16245 PGMACCESSORIGIN enmOrigin, void *pvUser)
16246{
16247 RT_NOREF3(pvPhys, enmOrigin, pvUser);
16248
16249 RTGCPHYS const GCPhysAccessBase = GCPhysFault & ~(RTGCPHYS)PAGE_OFFSET_MASK;
16250 if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
16251 {
16252 Assert(CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(pVCpu), VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16253 Assert(CPUMGetGuestVmxApicAccessPageAddr(IEM_GET_CTX(pVCpu)) == GCPhysAccessBase);
16254
16255 /** @todo NSTVMX: How are we to distinguish instruction fetch accesses here?
16256 * Currently they will go through as read accesses. */
16257 uint32_t const fAccess = enmAccessType == PGMACCESSTYPE_WRITE ? IEM_ACCESS_TYPE_WRITE : IEM_ACCESS_TYPE_READ;
16258 uint16_t const offAccess = GCPhysFault & PAGE_OFFSET_MASK;
16259 VBOXSTRICTRC rcStrict = iemVmxVirtApicAccessMem(pVCpu, offAccess, cbBuf, pvBuf, fAccess);
16260 if (RT_FAILURE(rcStrict))
16261 return rcStrict;
16262
16263 /* Any access on this APIC-access page has been handled, caller should not carry out the access. */
16264 return VINF_SUCCESS;
16265 }
16266
16267 Log(("iemVmxApicAccessPageHandler: Access outside VMX non-root mode, deregistering page at %#RGp\n", GCPhysAccessBase));
16268 int rc = PGMHandlerPhysicalDeregister(pVM, GCPhysAccessBase);
16269 if (RT_FAILURE(rc))
16270 return rc;
16271
16272 /* Instruct the caller of this handler to perform the read/write as normal memory. */
16273 return VINF_PGM_HANDLER_DO_DEFAULT;
16274}
16275
16276#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16277
16278#ifdef IN_RING3
16279
16280/**
16281 * Handles the unlikely and probably fatal merge cases.
16282 *
16283 * @returns Merged status code.
16284 * @param rcStrict Current EM status code.
16285 * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
16286 * with @a rcStrict.
16287 * @param iMemMap The memory mapping index. For error reporting only.
16288 * @param pVCpu The cross context virtual CPU structure of the calling
16289 * thread, for error reporting only.
16290 */
16291DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
16292 unsigned iMemMap, PVMCPUCC pVCpu)
16293{
16294 if (RT_FAILURE_NP(rcStrict))
16295 return rcStrict;
16296
16297 if (RT_FAILURE_NP(rcStrictCommit))
16298 return rcStrictCommit;
16299
16300 if (rcStrict == rcStrictCommit)
16301 return rcStrictCommit;
16302
16303 AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",
16304 VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,
16305 pVCpu->iem.s.aMemMappings[iMemMap].fAccess,
16306 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,
16307 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));
16308 return VERR_IOM_FF_STATUS_IPE;
16309}
16310
16311
16312/**
16313 * Helper for IOMR3ProcessForceFlag.
16314 *
16315 * @returns Merged status code.
16316 * @param rcStrict Current EM status code.
16317 * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
16318 * with @a rcStrict.
16319 * @param iMemMap The memory mapping index. For error reporting only.
16320 * @param pVCpu The cross context virtual CPU structure of the calling
16321 * thread, for error reporting only.
16322 */
16323DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu)
16324{
16325 /* Simple. */
16326 if (RT_LIKELY(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RAW_TO_R3))
16327 return rcStrictCommit;
16328
16329 if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
16330 return rcStrict;
16331
16332 /* EM scheduling status codes. */
16333 if (RT_LIKELY( rcStrict >= VINF_EM_FIRST
16334 && rcStrict <= VINF_EM_LAST))
16335 {
16336 if (RT_LIKELY( rcStrictCommit >= VINF_EM_FIRST
16337 && rcStrictCommit <= VINF_EM_LAST))
16338 return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
16339 }
16340
16341 /* Unlikely */
16342 return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);
16343}
16344
16345
16346/**
16347 * Called by force-flag handling code when VMCPU_FF_IEM is set.
16348 *
16349 * @returns Merge between @a rcStrict and what the commit operation returned.
16350 * @param pVM The cross context VM structure.
16351 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16352 * @param rcStrict The status code returned by ring-0 or raw-mode.
16353 */
16354VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
16355{
16356 /*
16357 * Reset the pending commit.
16358 */
16359 AssertMsg( (pVCpu->iem.s.aMemMappings[0].fAccess | pVCpu->iem.s.aMemMappings[1].fAccess | pVCpu->iem.s.aMemMappings[2].fAccess)
16360 & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND),
16361 ("%#x %#x %#x\n",
16362 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
16363 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);
16364
16365 /*
16366 * Commit the pending bounce buffers (usually just one).
16367 */
16368 unsigned cBufs = 0;
16369 unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
16370 while (iMemMap-- > 0)
16371 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND))
16372 {
16373 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
16374 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
16375 Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);
16376
16377 uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
16378 uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
16379 uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
16380
16381 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)
16382 {
16383 VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,
16384 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
16385 pbBuf,
16386 cbFirst,
16387 PGMACCESSORIGIN_IEM);
16388 rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);
16389 Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",
16390 iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
16391 VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));
16392 }
16393
16394 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)
16395 {
16396 VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,
16397 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
16398 pbBuf + cbFirst,
16399 cbSecond,
16400 PGMACCESSORIGIN_IEM);
16401 rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);
16402 Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",
16403 iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,
16404 VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));
16405 }
16406 cBufs++;
16407 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
16408 }
16409
16410 AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,
16411 ("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,
16412 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
16413 pVCpu->iem.s.cActiveMappings = 0;
16414 return rcStrict;
16415}
16416
16417#endif /* IN_RING3 */
16418
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette