IEMAll.cpp@ 40255

Last change on this file since 40255 was 40255, checked in by vboxsync, 13 years ago
Implemented fpu instruction stubs starting with 0xdd (fiadd m32i ++).
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 283.4 KB

Line
1	/* $Id: IEMAll.cpp 40255 2012-02-25 00:38:51Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2011-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/** @page pg_iem IEM - Interpreted Execution Manager
20	*
21	* The interpreted exeuction manager (IEM) is for executing short guest code
22	* sequences that are causing too many exits / virtualization traps. It will
23	* also be used to interpret single instructions, thus replacing the selective
24	* interpreters in EM and IOM.
25	*
26	* Design goals:
27	* - Relatively small footprint, although we favour speed and correctness
28	* over size.
29	* - Reasonably fast.
30	* - Correctly handle lock prefixed instructions.
31	* - Complete instruction set - eventually.
32	* - Refactorable into a recompiler, maybe.
33	* - Replace EMInterpret*.
34	*
35	* Using the existing disassembler has been considered, however this is thought
36	* to conflict with speed as the disassembler chews things a bit too much while
37	* leaving us with a somewhat complicated state to interpret afterwards.
38	*
39	*
40	* The current code is very much work in progress. You've been warned!
41	*
42	*
43	* @section sec_iem_fpu_instr FPU Instructions
44	*
45	* On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
46	* same or equivalent instructions on the host FPU. To make life easy, we also
47	* let the FPU prioritize the unmasked exceptions for us. This however, only
48	* works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
49	* for FPU exception delivery, because with CR0.NE=0 there is a window where we
50	* can trigger spurious FPU exceptions.
51	*
52	* The guest FPU state is not loaded into the host CPU and kept there till we
53	* leave IEM because the calling conventions have declared an all year open
54	* season on much of the FPU state. For instance an innocent looking call to
55	* memcpy might end up using a whole bunch of XMM or MM registers if the
56	* particular implementation finds it worthwhile.
57	*
58	*
59	* @section sec_iem_logging Logging
60	*
61	* The IEM code uses the \"IEM\" log group for the main logging. The different
62	* logging levels/flags are generally used for the following purposes:
63	* - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
64	* - Flow (LogFlow): Additional exception details, basic enter/exit IEM
65	* state info.
66	* - Level 2 (Log2): ?
67	* - Level 3 (Log3): More detailed enter/exit IEM state info.
68	* - Level 4 (Log4): Decoding mnemonics w/ EIP.
69	* - Level 5 (Log5): Decoding details.
70	* - Level 6 (Log6): Enables/disables the lockstep comparison with REM.
71	*
72	*/
73
74	/*******************************************************************************
75	* Header Files *
76	*******************************************************************************/
77	#define LOG_GROUP LOG_GROUP_IEM
78	#include <VBox/vmm/iem.h>
79	#include <VBox/vmm/pgm.h>
80	#include <VBox/vmm/iom.h>
81	#include <VBox/vmm/em.h>
82	#include <VBox/vmm/tm.h>
83	#include <VBox/vmm/dbgf.h>
84	#ifdef IEM_VERIFICATION_MODE
85	# include <VBox/vmm/rem.h>
86	# include <VBox/vmm/mm.h>
87	#endif
88	#include "IEMInternal.h"
89	#include <VBox/vmm/vm.h>
90	#include <VBox/log.h>
91	#include <VBox/err.h>
92	#include <VBox/param.h>
93	#include <iprt/assert.h>
94	#include <iprt/string.h>
95	#include <iprt/x86.h>
96
97
98	/*******************************************************************************
99	* Structures and Typedefs *
100	*******************************************************************************/
101	/** @typedef PFNIEMOP
102	* Pointer to an opcode decoder function.
103	*/
104
105	/** @def FNIEMOP_DEF
106	* Define an opcode decoder function.
107	*
108	* We're using macors for this so that adding and removing parameters as well as
109	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
110	*
111	* @param a_Name The function name.
112	*/
113
114
115	#if defined(__GNUC__) && defined(RT_ARCH_X86)
116	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PIEMCPU pIemCpu);
117	# define FNIEMOP_DEF(a_Name) \
118	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name (PIEMCPU pIemCpu)
119	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
120	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0)
121	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
122	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1)
123
124	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
125	typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PIEMCPU pIemCpu);
126	# define FNIEMOP_DEF(a_Name) \
127	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu) RT_NO_THROW
128	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
129	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0) RT_NO_THROW
130	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
131	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW
132
133	#elif defined(__GNUC__)
134	typedef VBOXSTRICTRC (* PFNIEMOP)(PIEMCPU pIemCpu);
135	# define FNIEMOP_DEF(a_Name) \
136	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu)
137	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
138	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0)
139	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
140	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1)
141
142	#else
143	typedef VBOXSTRICTRC (* PFNIEMOP)(PIEMCPU pIemCpu);
144	# define FNIEMOP_DEF(a_Name) \
145	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu) RT_NO_THROW
146	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
147	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0) RT_NO_THROW
148	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
149	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW
150
151	#endif
152
153
154	/**
155	* Selector descriptor table entry as fetched by iemMemFetchSelDesc.
156	*/
157	typedef union IEMSELDESC
158	{
159	/** The legacy view. */
160	X86DESC Legacy;
161	/** The long mode view. */
162	X86DESC64 Long;
163	} IEMSELDESC;
164	/** Pointer to a selector descriptor table entry. */
165	typedef IEMSELDESC *PIEMSELDESC;
166
167
168	/*******************************************************************************
169	* Defined Constants And Macros *
170	*******************************************************************************/
171	/** @name IEM status codes.
172	*
173	* Not quite sure how this will play out in the end, just aliasing safe status
174	* codes for now.
175	*
176	* @{ */
177	#define VINF_IEM_RAISED_XCPT VINF_EM_RESCHEDULE
178	/** @} */
179
180	/** Temporary hack to disable the double execution. Will be removed in favor
181	* of a dedicated execution mode in EM. */
182	//#define IEM_VERIFICATION_MODE_NO_REM
183
184	/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
185	* due to GCC lacking knowledge about the value range of a switch. */
186	#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
187
188	/**
189	* Call an opcode decoder function.
190	*
191	* We're using macors for this so that adding and removing parameters can be
192	* done as we please. See FNIEMOP_DEF.
193	*/
194	#define FNIEMOP_CALL(a_pfn) (a_pfn)(pIemCpu)
195
196	/**
197	* Call a common opcode decoder function taking one extra argument.
198	*
199	* We're using macors for this so that adding and removing parameters can be
200	* done as we please. See FNIEMOP_DEF_1.
201	*/
202	#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pIemCpu, a0)
203
204	/**
205	* Call a common opcode decoder function taking one extra argument.
206	*
207	* We're using macors for this so that adding and removing parameters can be
208	* done as we please. See FNIEMOP_DEF_1.
209	*/
210	#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pIemCpu, a0, a1)
211
212	/**
213	* Check if we're currently executing in real or virtual 8086 mode.
214	*
215	* @returns @c true if it is, @c false if not.
216	* @param a_pIemCpu The IEM state of the current CPU.
217	*/
218	#define IEM_IS_REAL_OR_V86_MODE(a_pIemCpu) (CPUMIsGuestInRealOrV86ModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
219
220	/**
221	* Check if we're currently executing in long mode.
222	*
223	* @returns @c true if it is, @c false if not.
224	* @param a_pIemCpu The IEM state of the current CPU.
225	*/
226	#define IEM_IS_LONG_MODE(a_pIemCpu) (CPUMIsGuestInLongModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
227
228	/**
229	* Check if we're currently executing in real mode.
230	*
231	* @returns @c true if it is, @c false if not.
232	* @param a_pIemCpu The IEM state of the current CPU.
233	*/
234	#define IEM_IS_REAL_MODE(a_pIemCpu) (CPUMIsGuestInRealModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
235
236	/**
237	* Tests if an AMD CPUID feature (extended) is marked present - ECX.
238	*/
239	#define IEM_IS_AMD_CPUID_FEATURE_PRESENT_ECX(a_fEcx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, 0, (a_fEcx))
240
241	/**
242	* Tests if an AMD CPUID feature (extended) is marked present - EDX.
243	*/
244	#define IEM_IS_AMD_CPUID_FEATURE_PRESENT_EDX(a_fEdx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, (a_fEdx), 0)
245
246	/**
247	* Tests if at least on of the specified AMD CPUID features (extended) are
248	* marked present.
249	*/
250	#define IEM_IS_AMD_CPUID_FEATURES_ANY_PRESENT(a_fEdx, a_fEcx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, (a_fEdx), (a_fEcx))
251
252	/**
253	* Checks if a intel CPUID feature is present.
254	*/
255	#define IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX(a_fEdx) \
256	( ((a_fEdx) & (X86_CPUID_FEATURE_EDX_TSC \| 0)) \
257	\|\| iemRegIsIntelCpuIdFeaturePresent(pIemCpu, (a_fEdx), 0) )
258
259	/**
260	* Check if the address is canonical.
261	*/
262	#define IEM_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
263
264
265	/*******************************************************************************
266	* Global Variables *
267	*******************************************************************************/
268	extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
269
270
271	/** Function table for the ADD instruction. */
272	static const IEMOPBINSIZES g_iemAImpl_add =
273	{
274	iemAImpl_add_u8, iemAImpl_add_u8_locked,
275	iemAImpl_add_u16, iemAImpl_add_u16_locked,
276	iemAImpl_add_u32, iemAImpl_add_u32_locked,
277	iemAImpl_add_u64, iemAImpl_add_u64_locked
278	};
279
280	/** Function table for the ADC instruction. */
281	static const IEMOPBINSIZES g_iemAImpl_adc =
282	{
283	iemAImpl_adc_u8, iemAImpl_adc_u8_locked,
284	iemAImpl_adc_u16, iemAImpl_adc_u16_locked,
285	iemAImpl_adc_u32, iemAImpl_adc_u32_locked,
286	iemAImpl_adc_u64, iemAImpl_adc_u64_locked
287	};
288
289	/** Function table for the SUB instruction. */
290	static const IEMOPBINSIZES g_iemAImpl_sub =
291	{
292	iemAImpl_sub_u8, iemAImpl_sub_u8_locked,
293	iemAImpl_sub_u16, iemAImpl_sub_u16_locked,
294	iemAImpl_sub_u32, iemAImpl_sub_u32_locked,
295	iemAImpl_sub_u64, iemAImpl_sub_u64_locked
296	};
297
298	/** Function table for the SBB instruction. */
299	static const IEMOPBINSIZES g_iemAImpl_sbb =
300	{
301	iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked,
302	iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked,
303	iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked,
304	iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked
305	};
306
307	/** Function table for the OR instruction. */
308	static const IEMOPBINSIZES g_iemAImpl_or =
309	{
310	iemAImpl_or_u8, iemAImpl_or_u8_locked,
311	iemAImpl_or_u16, iemAImpl_or_u16_locked,
312	iemAImpl_or_u32, iemAImpl_or_u32_locked,
313	iemAImpl_or_u64, iemAImpl_or_u64_locked
314	};
315
316	/** Function table for the XOR instruction. */
317	static const IEMOPBINSIZES g_iemAImpl_xor =
318	{
319	iemAImpl_xor_u8, iemAImpl_xor_u8_locked,
320	iemAImpl_xor_u16, iemAImpl_xor_u16_locked,
321	iemAImpl_xor_u32, iemAImpl_xor_u32_locked,
322	iemAImpl_xor_u64, iemAImpl_xor_u64_locked
323	};
324
325	/** Function table for the AND instruction. */
326	static const IEMOPBINSIZES g_iemAImpl_and =
327	{
328	iemAImpl_and_u8, iemAImpl_and_u8_locked,
329	iemAImpl_and_u16, iemAImpl_and_u16_locked,
330	iemAImpl_and_u32, iemAImpl_and_u32_locked,
331	iemAImpl_and_u64, iemAImpl_and_u64_locked
332	};
333
334	/** Function table for the CMP instruction.
335	* @remarks Making operand order ASSUMPTIONS.
336	*/
337	static const IEMOPBINSIZES g_iemAImpl_cmp =
338	{
339	iemAImpl_cmp_u8, NULL,
340	iemAImpl_cmp_u16, NULL,
341	iemAImpl_cmp_u32, NULL,
342	iemAImpl_cmp_u64, NULL
343	};
344
345	/** Function table for the TEST instruction.
346	* @remarks Making operand order ASSUMPTIONS.
347	*/
348	static const IEMOPBINSIZES g_iemAImpl_test =
349	{
350	iemAImpl_test_u8, NULL,
351	iemAImpl_test_u16, NULL,
352	iemAImpl_test_u32, NULL,
353	iemAImpl_test_u64, NULL
354	};
355
356	/** Function table for the BT instruction. */
357	static const IEMOPBINSIZES g_iemAImpl_bt =
358	{
359	NULL, NULL,
360	iemAImpl_bt_u16, NULL,
361	iemAImpl_bt_u32, NULL,
362	iemAImpl_bt_u64, NULL
363	};
364
365	/** Function table for the BTC instruction. */
366	static const IEMOPBINSIZES g_iemAImpl_btc =
367	{
368	NULL, NULL,
369	iemAImpl_btc_u16, iemAImpl_btc_u16_locked,
370	iemAImpl_btc_u32, iemAImpl_btc_u32_locked,
371	iemAImpl_btc_u64, iemAImpl_btc_u64_locked
372	};
373
374	/** Function table for the BTR instruction. */
375	static const IEMOPBINSIZES g_iemAImpl_btr =
376	{
377	NULL, NULL,
378	iemAImpl_btr_u16, iemAImpl_btr_u16_locked,
379	iemAImpl_btr_u32, iemAImpl_btr_u32_locked,
380	iemAImpl_btr_u64, iemAImpl_btr_u64_locked
381	};
382
383	/** Function table for the BTS instruction. */
384	static const IEMOPBINSIZES g_iemAImpl_bts =
385	{
386	NULL, NULL,
387	iemAImpl_bts_u16, iemAImpl_bts_u16_locked,
388	iemAImpl_bts_u32, iemAImpl_bts_u32_locked,
389	iemAImpl_bts_u64, iemAImpl_bts_u64_locked
390	};
391
392	/** Function table for the BSF instruction. */
393	static const IEMOPBINSIZES g_iemAImpl_bsf =
394	{
395	NULL, NULL,
396	iemAImpl_bsf_u16, NULL,
397	iemAImpl_bsf_u32, NULL,
398	iemAImpl_bsf_u64, NULL
399	};
400
401	/** Function table for the BSR instruction. */
402	static const IEMOPBINSIZES g_iemAImpl_bsr =
403	{
404	NULL, NULL,
405	iemAImpl_bsr_u16, NULL,
406	iemAImpl_bsr_u32, NULL,
407	iemAImpl_bsr_u64, NULL
408	};
409
410	/** Function table for the IMUL instruction. */
411	static const IEMOPBINSIZES g_iemAImpl_imul_two =
412	{
413	NULL, NULL,
414	iemAImpl_imul_two_u16, NULL,
415	iemAImpl_imul_two_u32, NULL,
416	iemAImpl_imul_two_u64, NULL
417	};
418
419	/** Group 1 /r lookup table. */
420	static const PCIEMOPBINSIZES g_apIemImplGrp1[8] =
421	{
422	&g_iemAImpl_add,
423	&g_iemAImpl_or,
424	&g_iemAImpl_adc,
425	&g_iemAImpl_sbb,
426	&g_iemAImpl_and,
427	&g_iemAImpl_sub,
428	&g_iemAImpl_xor,
429	&g_iemAImpl_cmp
430	};
431
432	/** Function table for the INC instruction. */
433	static const IEMOPUNARYSIZES g_iemAImpl_inc =
434	{
435	iemAImpl_inc_u8, iemAImpl_inc_u8_locked,
436	iemAImpl_inc_u16, iemAImpl_inc_u16_locked,
437	iemAImpl_inc_u32, iemAImpl_inc_u32_locked,
438	iemAImpl_inc_u64, iemAImpl_inc_u64_locked
439	};
440
441	/** Function table for the DEC instruction. */
442	static const IEMOPUNARYSIZES g_iemAImpl_dec =
443	{
444	iemAImpl_dec_u8, iemAImpl_dec_u8_locked,
445	iemAImpl_dec_u16, iemAImpl_dec_u16_locked,
446	iemAImpl_dec_u32, iemAImpl_dec_u32_locked,
447	iemAImpl_dec_u64, iemAImpl_dec_u64_locked
448	};
449
450	/** Function table for the NEG instruction. */
451	static const IEMOPUNARYSIZES g_iemAImpl_neg =
452	{
453	iemAImpl_neg_u8, iemAImpl_neg_u8_locked,
454	iemAImpl_neg_u16, iemAImpl_neg_u16_locked,
455	iemAImpl_neg_u32, iemAImpl_neg_u32_locked,
456	iemAImpl_neg_u64, iemAImpl_neg_u64_locked
457	};
458
459	/** Function table for the NOT instruction. */
460	static const IEMOPUNARYSIZES g_iemAImpl_not =
461	{
462	iemAImpl_not_u8, iemAImpl_not_u8_locked,
463	iemAImpl_not_u16, iemAImpl_not_u16_locked,
464	iemAImpl_not_u32, iemAImpl_not_u32_locked,
465	iemAImpl_not_u64, iemAImpl_not_u64_locked
466	};
467
468
469	/** Function table for the ROL instruction. */
470	static const IEMOPSHIFTSIZES g_iemAImpl_rol =
471	{
472	iemAImpl_rol_u8,
473	iemAImpl_rol_u16,
474	iemAImpl_rol_u32,
475	iemAImpl_rol_u64
476	};
477
478	/** Function table for the ROR instruction. */
479	static const IEMOPSHIFTSIZES g_iemAImpl_ror =
480	{
481	iemAImpl_ror_u8,
482	iemAImpl_ror_u16,
483	iemAImpl_ror_u32,
484	iemAImpl_ror_u64
485	};
486
487	/** Function table for the RCL instruction. */
488	static const IEMOPSHIFTSIZES g_iemAImpl_rcl =
489	{
490	iemAImpl_rcl_u8,
491	iemAImpl_rcl_u16,
492	iemAImpl_rcl_u32,
493	iemAImpl_rcl_u64
494	};
495
496	/** Function table for the RCR instruction. */
497	static const IEMOPSHIFTSIZES g_iemAImpl_rcr =
498	{
499	iemAImpl_rcr_u8,
500	iemAImpl_rcr_u16,
501	iemAImpl_rcr_u32,
502	iemAImpl_rcr_u64
503	};
504
505	/** Function table for the SHL instruction. */
506	static const IEMOPSHIFTSIZES g_iemAImpl_shl =
507	{
508	iemAImpl_shl_u8,
509	iemAImpl_shl_u16,
510	iemAImpl_shl_u32,
511	iemAImpl_shl_u64
512	};
513
514	/** Function table for the SHR instruction. */
515	static const IEMOPSHIFTSIZES g_iemAImpl_shr =
516	{
517	iemAImpl_shr_u8,
518	iemAImpl_shr_u16,
519	iemAImpl_shr_u32,
520	iemAImpl_shr_u64
521	};
522
523	/** Function table for the SAR instruction. */
524	static const IEMOPSHIFTSIZES g_iemAImpl_sar =
525	{
526	iemAImpl_sar_u8,
527	iemAImpl_sar_u16,
528	iemAImpl_sar_u32,
529	iemAImpl_sar_u64
530	};
531
532
533	/** Function table for the MUL instruction. */
534	static const IEMOPMULDIVSIZES g_iemAImpl_mul =
535	{
536	iemAImpl_mul_u8,
537	iemAImpl_mul_u16,
538	iemAImpl_mul_u32,
539	iemAImpl_mul_u64
540	};
541
542	/** Function table for the IMUL instruction working implicitly on rAX. */
543	static const IEMOPMULDIVSIZES g_iemAImpl_imul =
544	{
545	iemAImpl_imul_u8,
546	iemAImpl_imul_u16,
547	iemAImpl_imul_u32,
548	iemAImpl_imul_u64
549	};
550
551	/** Function table for the DIV instruction. */
552	static const IEMOPMULDIVSIZES g_iemAImpl_div =
553	{
554	iemAImpl_div_u8,
555	iemAImpl_div_u16,
556	iemAImpl_div_u32,
557	iemAImpl_div_u64
558	};
559
560	/** Function table for the MUL instruction. */
561	static const IEMOPMULDIVSIZES g_iemAImpl_idiv =
562	{
563	iemAImpl_idiv_u8,
564	iemAImpl_idiv_u16,
565	iemAImpl_idiv_u32,
566	iemAImpl_idiv_u64
567	};
568
569	/** Function table for the SHLD instruction */
570	static const IEMOPSHIFTDBLSIZES g_iemAImpl_shld =
571	{
572	iemAImpl_shld_u16,
573	iemAImpl_shld_u32,
574	iemAImpl_shld_u64,
575	};
576
577	/** Function table for the SHRD instruction */
578	static const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd =
579	{
580	iemAImpl_shrd_u16,
581	iemAImpl_shrd_u32,
582	iemAImpl_shrd_u64,
583	};
584
585
586	/*******************************************************************************
587	* Internal Functions *
588	*******************************************************************************/
589	static VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PIEMCPU pIemCpu);
590	/static VBOXSTRICTRC iemRaiseSelectorNotPresent(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);/
591	static VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel);
592	static VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PIEMCPU pIemCpu, uint16_t uErr);
593	static VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel);
594	static VBOXSTRICTRC iemRaiseGeneralProtectionFault(PIEMCPU pIemCpu, uint16_t uErr);
595	static VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PIEMCPU pIemCpu);
596	static VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PIEMCPU pIemCpu, RTSEL uSel);
597	static VBOXSTRICTRC iemRaiseSelectorBounds(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);
598	static VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PIEMCPU pIemCpu, RTSEL Sel);
599	static VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);
600	static VBOXSTRICTRC iemRaisePageFault(PIEMCPU pIemCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
601	static VBOXSTRICTRC iemRaiseAlignmentCheckException(PIEMCPU pIemCpu);
602	static VBOXSTRICTRC iemMemMap(PIEMCPU pIemCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess);
603	static VBOXSTRICTRC iemMemCommitAndUnmap(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess);
604	static VBOXSTRICTRC iemMemFetchDataU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
605	static VBOXSTRICTRC iemMemFetchDataU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
606	static VBOXSTRICTRC iemMemFetchSysU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
607	static VBOXSTRICTRC iemMemFetchSysU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
608	static VBOXSTRICTRC iemMemFetchSelDesc(PIEMCPU pIemCpu, PIEMSELDESC pDesc, uint16_t uSel);
609	static VBOXSTRICTRC iemMemStackPushCommitSpecial(PIEMCPU pIemCpu, void *pvMem, uint64_t uNewRsp);
610	static VBOXSTRICTRC iemMemStackPushBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void *ppvMem, uint64_t puNewRsp);
611	static VBOXSTRICTRC iemMemMarkSelDescAccessed(PIEMCPU pIemCpu, uint16_t uSel);
612	static uint16_t iemSRegFetchU16(PIEMCPU pIemCpu, uint8_t iSegReg);
613
614	#ifdef IEM_VERIFICATION_MODE
615	static PIEMVERIFYEVTREC iemVerifyAllocRecord(PIEMCPU pIemCpu);
616	#endif
617	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue);
618	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue);
619
620
621	/**
622	* Initializes the decoder state.
623	*
624	* @param pIemCpu The per CPU IEM state.
625	*/
626	DECLINLINE(void) iemInitDecoder(PIEMCPU pIemCpu)
627	{
628	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
629
630	pIemCpu->uCpl = CPUMGetGuestCPL(IEMCPU_TO_VMCPU(pIemCpu), CPUMCTX2CORE(pCtx));
631	IEMMODE enmMode = CPUMIsGuestIn64BitCodeEx(pCtx)
632	? IEMMODE_64BIT
633	: pCtx->csHid.Attr.n.u1DefBig /** @todo check if this is correct... */
634	? IEMMODE_32BIT
635	: IEMMODE_16BIT;
636	pIemCpu->enmCpuMode = enmMode;
637	pIemCpu->enmDefAddrMode = enmMode; /** @todo check if this is correct... */
638	pIemCpu->enmEffAddrMode = enmMode;
639	pIemCpu->enmDefOpSize = enmMode; /** @todo check if this is correct... */
640	pIemCpu->enmEffOpSize = enmMode;
641	pIemCpu->fPrefixes = 0;
642	pIemCpu->uRexReg = 0;
643	pIemCpu->uRexB = 0;
644	pIemCpu->uRexIndex = 0;
645	pIemCpu->iEffSeg = X86_SREG_DS;
646	pIemCpu->offOpcode = 0;
647	pIemCpu->cbOpcode = 0;
648	pIemCpu->cActiveMappings = 0;
649	pIemCpu->iNextMapping = 0;
650	}
651
652
653	/**
654	* Prefetch opcodes the first time when starting executing.
655	*
656	* @returns Strict VBox status code.
657	* @param pIemCpu The IEM state.
658	*/
659	static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PIEMCPU pIemCpu)
660	{
661	#ifdef IEM_VERIFICATION_MODE
662	uint8_t const cbOldOpcodes = pIemCpu->cbOpcode;
663	#endif
664	iemInitDecoder(pIemCpu);
665
666	/*
667	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
668	*
669	* First translate CS:rIP to a physical address.
670	*/
671	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
672	uint32_t cbToTryRead;
673	RTGCPTR GCPtrPC;
674	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
675	{
676	cbToTryRead = PAGE_SIZE;
677	GCPtrPC = pCtx->rip;
678	if (!IEM_IS_CANONICAL(GCPtrPC))
679	return iemRaiseGeneralProtectionFault0(pIemCpu);
680	cbToTryRead = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
681	}
682	else
683	{
684	uint32_t GCPtrPC32 = pCtx->eip;
685	Assert(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) \|\| pIemCpu->enmCpuMode == IEMMODE_32BIT);
686	if (GCPtrPC32 > pCtx->csHid.u32Limit)
687	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
688	cbToTryRead = pCtx->csHid.u32Limit - GCPtrPC32 + 1;
689	GCPtrPC = pCtx->csHid.u64Base + GCPtrPC32;
690	}
691
692	RTGCPHYS GCPhys;
693	uint64_t fFlags;
694	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrPC, &fFlags, &GCPhys);
695	if (RT_FAILURE(rc))
696	{
697	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
698	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
699	}
700	if (!(fFlags & X86_PTE_US) && pIemCpu->uCpl == 3)
701	{
702	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
703	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
704	}
705	if ((fFlags & X86_PTE_PAE_NX) && (pCtx->msrEFER & MSR_K6_EFER_NXE))
706	{
707	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
708	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
709	}
710	GCPhys \|= GCPtrPC & PAGE_OFFSET_MASK;
711	/** @todo Check reserved bits and such stuff. PGM is better at doing
712	* that, so do it when implementing the guest virtual address
713	* TLB... */
714
715	#ifdef IEM_VERIFICATION_MODE
716	/*
717	* Optimistic optimization: Use unconsumed opcode bytes from the previous
718	* instruction.
719	*/
720	/** @todo optimize this differently by not using PGMPhysRead. */
721	RTGCPHYS const offPrevOpcodes = GCPhys - pIemCpu->GCPhysOpcodes;
722	pIemCpu->GCPhysOpcodes = GCPhys;
723	if ( offPrevOpcodes < cbOldOpcodes
724	&& PAGE_SIZE - (GCPhys & PAGE_OFFSET_MASK) > sizeof(pIemCpu->abOpcode))
725	{
726	uint8_t cbNew = cbOldOpcodes - (uint8_t)offPrevOpcodes;
727	memmove(&pIemCpu->abOpcode[0], &pIemCpu->abOpcode[offPrevOpcodes], cbNew);
728	pIemCpu->cbOpcode = cbNew;
729	return VINF_SUCCESS;
730	}
731	#endif
732
733	/*
734	* Read the bytes at this address.
735	*/
736	uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
737	if (cbToTryRead > cbLeftOnPage)
738	cbToTryRead = cbLeftOnPage;
739	if (cbToTryRead > sizeof(pIemCpu->abOpcode))
740	cbToTryRead = sizeof(pIemCpu->abOpcode);
741	/** @todo patch manager */
742	if (!pIemCpu->fByPassHandlers)
743	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhys, pIemCpu->abOpcode, cbToTryRead);
744	else
745	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pIemCpu->abOpcode, GCPhys, cbToTryRead);
746	if (rc != VINF_SUCCESS)
747	{
748	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - read error - rc=%Rrc\n", GCPtrPC, rc));
749	return rc;
750	}
751	pIemCpu->cbOpcode = cbToTryRead;
752
753	return VINF_SUCCESS;
754	}
755
756
757	/**
758	* Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
759	* exception if it fails.
760	*
761	* @returns Strict VBox status code.
762	* @param pIemCpu The IEM state.
763	* @param cbMin Where to return the opcode byte.
764	*/
765	static VBOXSTRICTRC iemOpcodeFetchMoreBytes(PIEMCPU pIemCpu, size_t cbMin)
766	{
767	/*
768	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
769	*
770	* First translate CS:rIP to a physical address.
771	*/
772	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
773	uint8_t cbLeft = pIemCpu->cbOpcode - pIemCpu->offOpcode; Assert(cbLeft < cbMin);
774	uint32_t cbToTryRead;
775	RTGCPTR GCPtrNext;
776	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
777	{
778	cbToTryRead = PAGE_SIZE;
779	GCPtrNext = pCtx->rip + pIemCpu->cbOpcode;
780	if (!IEM_IS_CANONICAL(GCPtrNext))
781	return iemRaiseGeneralProtectionFault0(pIemCpu);
782	cbToTryRead = PAGE_SIZE - (GCPtrNext & PAGE_OFFSET_MASK);
783	Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
784	}
785	else
786	{
787	uint32_t GCPtrNext32 = pCtx->eip;
788	Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) \|\| pIemCpu->enmCpuMode == IEMMODE_32BIT);
789	GCPtrNext32 += pIemCpu->cbOpcode;
790	if (GCPtrNext32 > pCtx->csHid.u32Limit)
791	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
792	cbToTryRead = pCtx->csHid.u32Limit - GCPtrNext32 + 1;
793	if (cbToTryRead < cbMin - cbLeft)
794	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
795	GCPtrNext = pCtx->csHid.u64Base + GCPtrNext32;
796	}
797
798	RTGCPHYS GCPhys;
799	uint64_t fFlags;
800	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrNext, &fFlags, &GCPhys);
801	if (RT_FAILURE(rc))
802	{
803	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
804	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
805	}
806	if (!(fFlags & X86_PTE_US) && pIemCpu->uCpl == 3)
807	{
808	Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
809	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
810	}
811	if ((fFlags & X86_PTE_PAE_NX) && (pCtx->msrEFER & MSR_K6_EFER_NXE))
812	{
813	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
814	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
815	}
816	GCPhys \|= GCPtrNext & PAGE_OFFSET_MASK;
817	Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pIemCpu->cbOpcode));
818	/** @todo Check reserved bits and such stuff. PGM is better at doing
819	* that, so do it when implementing the guest virtual address
820	* TLB... */
821
822	/*
823	* Read the bytes at this address.
824	*/
825	uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrNext & PAGE_OFFSET_MASK);
826	if (cbToTryRead > cbLeftOnPage)
827	cbToTryRead = cbLeftOnPage;
828	if (cbToTryRead > sizeof(pIemCpu->abOpcode) - pIemCpu->cbOpcode)
829	cbToTryRead = sizeof(pIemCpu->abOpcode) - pIemCpu->cbOpcode;
830	Assert(cbToTryRead >= cbMin - cbLeft);
831	if (!pIemCpu->fByPassHandlers)
832	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhys, &pIemCpu->abOpcode[pIemCpu->cbOpcode], cbToTryRead);
833	else
834	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), &pIemCpu->abOpcode[pIemCpu->cbOpcode], GCPhys, cbToTryRead);
835	if (rc != VINF_SUCCESS)
836	{
837	Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc\n", GCPtrNext, rc));
838	return rc;
839	}
840	pIemCpu->cbOpcode += cbToTryRead;
841	Log5(("%.*Rhxs\n", pIemCpu->cbOpcode, pIemCpu->abOpcode));
842
843	return VINF_SUCCESS;
844	}
845
846
847	/**
848	* Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
849	*
850	* @returns Strict VBox status code.
851	* @param pIemCpu The IEM state.
852	* @param pb Where to return the opcode byte.
853	*/
854	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU8Slow(PIEMCPU pIemCpu, uint8_t *pb)
855	{
856	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 1);
857	if (rcStrict == VINF_SUCCESS)
858	{
859	uint8_t offOpcode = pIemCpu->offOpcode;
860	*pb = pIemCpu->abOpcode[offOpcode];
861	pIemCpu->offOpcode = offOpcode + 1;
862	}
863	else
864	*pb = 0;
865	return rcStrict;
866	}
867
868
869	/**
870	* Fetches the next opcode byte.
871	*
872	* @returns Strict VBox status code.
873	* @param pIemCpu The IEM state.
874	* @param pu8 Where to return the opcode byte.
875	*/
876	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PIEMCPU pIemCpu, uint8_t *pu8)
877	{
878	uint8_t const offOpcode = pIemCpu->offOpcode;
879	if (RT_UNLIKELY(offOpcode >= pIemCpu->cbOpcode))
880	return iemOpcodeGetNextU8Slow(pIemCpu, pu8);
881
882	*pu8 = pIemCpu->abOpcode[offOpcode];
883	pIemCpu->offOpcode = offOpcode + 1;
884	return VINF_SUCCESS;
885	}
886
887
888	/**
889	* Fetches the next opcode byte, returns automatically on failure.
890	*
891	* @param a_pu8 Where to return the opcode byte.
892	* @remark Implicitly references pIemCpu.
893	*/
894	#define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
895	do \
896	{ \
897	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pIemCpu, (a_pu8)); \
898	if (rcStrict2 != VINF_SUCCESS) \
899	return rcStrict2; \
900	} while (0)
901
902
903	/**
904	* Fetches the next signed byte from the opcode stream.
905	*
906	* @returns Strict VBox status code.
907	* @param pIemCpu The IEM state.
908	* @param pi8 Where to return the signed byte.
909	*/
910	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PIEMCPU pIemCpu, int8_t *pi8)
911	{
912	return iemOpcodeGetNextU8(pIemCpu, (uint8_t *)pi8);
913	}
914
915
916	/**
917	* Fetches the next signed byte from the opcode stream, returning automatically
918	* on failure.
919	*
920	* @param pi8 Where to return the signed byte.
921	* @remark Implicitly references pIemCpu.
922	*/
923	#define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
924	do \
925	{ \
926	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pIemCpu, (a_pi8)); \
927	if (rcStrict2 != VINF_SUCCESS) \
928	return rcStrict2; \
929	} while (0)
930
931
932	/**
933	* Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
934	*
935	* @returns Strict VBox status code.
936	* @param pIemCpu The IEM state.
937	* @param pu16 Where to return the opcode dword.
938	*/
939	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextS8SxU16Slow(PIEMCPU pIemCpu, uint16_t *pu16)
940	{
941	uint8_t u8;
942	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pIemCpu, &u8);
943	if (rcStrict == VINF_SUCCESS)
944	*pu16 = (int8_t)u8;
945	return rcStrict;
946	}
947
948
949	/**
950	* Fetches the next signed byte from the opcode stream, extending it to
951	* unsigned 16-bit.
952	*
953	* @returns Strict VBox status code.
954	* @param pIemCpu The IEM state.
955	* @param pu16 Where to return the unsigned word.
956	*/
957	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PIEMCPU pIemCpu, uint16_t *pu16)
958	{
959	uint8_t const offOpcode = pIemCpu->offOpcode;
960	if (RT_UNLIKELY(offOpcode >= pIemCpu->cbOpcode))
961	return iemOpcodeGetNextS8SxU16Slow(pIemCpu, pu16);
962
963	*pu16 = (int8_t)pIemCpu->abOpcode[offOpcode];
964	pIemCpu->offOpcode = offOpcode + 1;
965	return VINF_SUCCESS;
966	}
967
968
969	/**
970	* Fetches the next signed byte from the opcode stream and sign-extending it to
971	* a word, returning automatically on failure.
972	*
973	* @param pu16 Where to return the word.
974	* @remark Implicitly references pIemCpu.
975	*/
976	#define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
977	do \
978	{ \
979	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pIemCpu, (a_pu16)); \
980	if (rcStrict2 != VINF_SUCCESS) \
981	return rcStrict2; \
982	} while (0)
983
984
985	/**
986	* Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
987	*
988	* @returns Strict VBox status code.
989	* @param pIemCpu The IEM state.
990	* @param pu16 Where to return the opcode word.
991	*/
992	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16Slow(PIEMCPU pIemCpu, uint16_t *pu16)
993	{
994	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
995	if (rcStrict == VINF_SUCCESS)
996	{
997	uint8_t offOpcode = pIemCpu->offOpcode;
998	*pu16 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
999	pIemCpu->offOpcode = offOpcode + 2;
1000	}
1001	else
1002	*pu16 = 0;
1003	return rcStrict;
1004	}
1005
1006
1007	/**
1008	* Fetches the next opcode word.
1009	*
1010	* @returns Strict VBox status code.
1011	* @param pIemCpu The IEM state.
1012	* @param pu16 Where to return the opcode word.
1013	*/
1014	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PIEMCPU pIemCpu, uint16_t *pu16)
1015	{
1016	uint8_t const offOpcode = pIemCpu->offOpcode;
1017	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1018	return iemOpcodeGetNextU16Slow(pIemCpu, pu16);
1019
1020	*pu16 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1021	pIemCpu->offOpcode = offOpcode + 2;
1022	return VINF_SUCCESS;
1023	}
1024
1025
1026	/**
1027	* Fetches the next opcode word, returns automatically on failure.
1028	*
1029	* @param a_pu16 Where to return the opcode word.
1030	* @remark Implicitly references pIemCpu.
1031	*/
1032	#define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
1033	do \
1034	{ \
1035	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pIemCpu, (a_pu16)); \
1036	if (rcStrict2 != VINF_SUCCESS) \
1037	return rcStrict2; \
1038	} while (0)
1039
1040
1041	/**
1042	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
1043	*
1044	* @returns Strict VBox status code.
1045	* @param pIemCpu The IEM state.
1046	* @param pu32 Where to return the opcode double word.
1047	*/
1048	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32Slow(PIEMCPU pIemCpu, uint32_t *pu32)
1049	{
1050	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
1051	if (rcStrict == VINF_SUCCESS)
1052	{
1053	uint8_t offOpcode = pIemCpu->offOpcode;
1054	*pu32 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1055	pIemCpu->offOpcode = offOpcode + 2;
1056	}
1057	else
1058	*pu32 = 0;
1059	return rcStrict;
1060	}
1061
1062
1063	/**
1064	* Fetches the next opcode word, zero extending it to a double word.
1065	*
1066	* @returns Strict VBox status code.
1067	* @param pIemCpu The IEM state.
1068	* @param pu32 Where to return the opcode double word.
1069	*/
1070	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PIEMCPU pIemCpu, uint32_t *pu32)
1071	{
1072	uint8_t const offOpcode = pIemCpu->offOpcode;
1073	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1074	return iemOpcodeGetNextU16ZxU32Slow(pIemCpu, pu32);
1075
1076	*pu32 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1077	pIemCpu->offOpcode = offOpcode + 2;
1078	return VINF_SUCCESS;
1079	}
1080
1081
1082	/**
1083	* Fetches the next opcode word and zero extends it to a double word, returns
1084	* automatically on failure.
1085	*
1086	* @param a_pu32 Where to return the opcode double word.
1087	* @remark Implicitly references pIemCpu.
1088	*/
1089	#define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1090	do \
1091	{ \
1092	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pIemCpu, (a_pu32)); \
1093	if (rcStrict2 != VINF_SUCCESS) \
1094	return rcStrict2; \
1095	} while (0)
1096
1097
1098	/**
1099	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
1100	*
1101	* @returns Strict VBox status code.
1102	* @param pIemCpu The IEM state.
1103	* @param pu64 Where to return the opcode quad word.
1104	*/
1105	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1106	{
1107	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
1108	if (rcStrict == VINF_SUCCESS)
1109	{
1110	uint8_t offOpcode = pIemCpu->offOpcode;
1111	*pu64 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1112	pIemCpu->offOpcode = offOpcode + 2;
1113	}
1114	else
1115	*pu64 = 0;
1116	return rcStrict;
1117	}
1118
1119
1120	/**
1121	* Fetches the next opcode word, zero extending it to a quad word.
1122	*
1123	* @returns Strict VBox status code.
1124	* @param pIemCpu The IEM state.
1125	* @param pu64 Where to return the opcode quad word.
1126	*/
1127	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1128	{
1129	uint8_t const offOpcode = pIemCpu->offOpcode;
1130	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1131	return iemOpcodeGetNextU16ZxU64Slow(pIemCpu, pu64);
1132
1133	*pu64 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1134	pIemCpu->offOpcode = offOpcode + 2;
1135	return VINF_SUCCESS;
1136	}
1137
1138
1139	/**
1140	* Fetches the next opcode word and zero extends it to a quad word, returns
1141	* automatically on failure.
1142	*
1143	* @param a_pu64 Where to return the opcode quad word.
1144	* @remark Implicitly references pIemCpu.
1145	*/
1146	#define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1147	do \
1148	{ \
1149	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pIemCpu, (a_pu64)); \
1150	if (rcStrict2 != VINF_SUCCESS) \
1151	return rcStrict2; \
1152	} while (0)
1153
1154
1155	/**
1156	* Fetches the next signed word from the opcode stream.
1157	*
1158	* @returns Strict VBox status code.
1159	* @param pIemCpu The IEM state.
1160	* @param pi16 Where to return the signed word.
1161	*/
1162	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PIEMCPU pIemCpu, int16_t *pi16)
1163	{
1164	return iemOpcodeGetNextU16(pIemCpu, (uint16_t *)pi16);
1165	}
1166
1167
1168	/**
1169	* Fetches the next signed word from the opcode stream, returning automatically
1170	* on failure.
1171	*
1172	* @param pi16 Where to return the signed word.
1173	* @remark Implicitly references pIemCpu.
1174	*/
1175	#define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1176	do \
1177	{ \
1178	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pIemCpu, (a_pi16)); \
1179	if (rcStrict2 != VINF_SUCCESS) \
1180	return rcStrict2; \
1181	} while (0)
1182
1183
1184	/**
1185	* Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
1186	*
1187	* @returns Strict VBox status code.
1188	* @param pIemCpu The IEM state.
1189	* @param pu32 Where to return the opcode dword.
1190	*/
1191	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU32Slow(PIEMCPU pIemCpu, uint32_t *pu32)
1192	{
1193	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1194	if (rcStrict == VINF_SUCCESS)
1195	{
1196	uint8_t offOpcode = pIemCpu->offOpcode;
1197	*pu32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1198	pIemCpu->abOpcode[offOpcode + 1],
1199	pIemCpu->abOpcode[offOpcode + 2],
1200	pIemCpu->abOpcode[offOpcode + 3]);
1201	pIemCpu->offOpcode = offOpcode + 4;
1202	}
1203	else
1204	*pu32 = 0;
1205	return rcStrict;
1206	}
1207
1208
1209	/**
1210	* Fetches the next opcode dword.
1211	*
1212	* @returns Strict VBox status code.
1213	* @param pIemCpu The IEM state.
1214	* @param pu32 Where to return the opcode double word.
1215	*/
1216	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PIEMCPU pIemCpu, uint32_t *pu32)
1217	{
1218	uint8_t const offOpcode = pIemCpu->offOpcode;
1219	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1220	return iemOpcodeGetNextU32Slow(pIemCpu, pu32);
1221
1222	*pu32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1223	pIemCpu->abOpcode[offOpcode + 1],
1224	pIemCpu->abOpcode[offOpcode + 2],
1225	pIemCpu->abOpcode[offOpcode + 3]);
1226	pIemCpu->offOpcode = offOpcode + 4;
1227	return VINF_SUCCESS;
1228	}
1229
1230
1231	/**
1232	* Fetches the next opcode dword, returns automatically on failure.
1233	*
1234	* @param a_pu32 Where to return the opcode dword.
1235	* @remark Implicitly references pIemCpu.
1236	*/
1237	#define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1238	do \
1239	{ \
1240	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pIemCpu, (a_pu32)); \
1241	if (rcStrict2 != VINF_SUCCESS) \
1242	return rcStrict2; \
1243	} while (0)
1244
1245
1246	/**
1247	* Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
1248	*
1249	* @returns Strict VBox status code.
1250	* @param pIemCpu The IEM state.
1251	* @param pu32 Where to return the opcode dword.
1252	*/
1253	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1254	{
1255	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1256	if (rcStrict == VINF_SUCCESS)
1257	{
1258	uint8_t offOpcode = pIemCpu->offOpcode;
1259	*pu64 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1260	pIemCpu->abOpcode[offOpcode + 1],
1261	pIemCpu->abOpcode[offOpcode + 2],
1262	pIemCpu->abOpcode[offOpcode + 3]);
1263	pIemCpu->offOpcode = offOpcode + 4;
1264	}
1265	else
1266	*pu64 = 0;
1267	return rcStrict;
1268	}
1269
1270
1271	/**
1272	* Fetches the next opcode dword, zero extending it to a quad word.
1273	*
1274	* @returns Strict VBox status code.
1275	* @param pIemCpu The IEM state.
1276	* @param pu64 Where to return the opcode quad word.
1277	*/
1278	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1279	{
1280	uint8_t const offOpcode = pIemCpu->offOpcode;
1281	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1282	return iemOpcodeGetNextU32ZxU64Slow(pIemCpu, pu64);
1283
1284	*pu64 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1285	pIemCpu->abOpcode[offOpcode + 1],
1286	pIemCpu->abOpcode[offOpcode + 2],
1287	pIemCpu->abOpcode[offOpcode + 3]);
1288	pIemCpu->offOpcode = offOpcode + 4;
1289	return VINF_SUCCESS;
1290	}
1291
1292
1293	/**
1294	* Fetches the next opcode dword and zero extends it to a quad word, returns
1295	* automatically on failure.
1296	*
1297	* @param a_pu64 Where to return the opcode quad word.
1298	* @remark Implicitly references pIemCpu.
1299	*/
1300	#define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1301	do \
1302	{ \
1303	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pIemCpu, (a_pu64)); \
1304	if (rcStrict2 != VINF_SUCCESS) \
1305	return rcStrict2; \
1306	} while (0)
1307
1308
1309	/**
1310	* Fetches the next signed double word from the opcode stream.
1311	*
1312	* @returns Strict VBox status code.
1313	* @param pIemCpu The IEM state.
1314	* @param pi32 Where to return the signed double word.
1315	*/
1316	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PIEMCPU pIemCpu, int32_t *pi32)
1317	{
1318	return iemOpcodeGetNextU32(pIemCpu, (uint32_t *)pi32);
1319	}
1320
1321	/**
1322	* Fetches the next signed double word from the opcode stream, returning
1323	* automatically on failure.
1324	*
1325	* @param pi32 Where to return the signed double word.
1326	* @remark Implicitly references pIemCpu.
1327	*/
1328	#define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1329	do \
1330	{ \
1331	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pIemCpu, (a_pi32)); \
1332	if (rcStrict2 != VINF_SUCCESS) \
1333	return rcStrict2; \
1334	} while (0)
1335
1336
1337	/**
1338	* Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
1339	*
1340	* @returns Strict VBox status code.
1341	* @param pIemCpu The IEM state.
1342	* @param pu64 Where to return the opcode qword.
1343	*/
1344	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextS32SxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1345	{
1346	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1347	if (rcStrict == VINF_SUCCESS)
1348	{
1349	uint8_t offOpcode = pIemCpu->offOpcode;
1350	*pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1351	pIemCpu->abOpcode[offOpcode + 1],
1352	pIemCpu->abOpcode[offOpcode + 2],
1353	pIemCpu->abOpcode[offOpcode + 3]);
1354	pIemCpu->offOpcode = offOpcode + 4;
1355	}
1356	else
1357	*pu64 = 0;
1358	return rcStrict;
1359	}
1360
1361
1362	/**
1363	* Fetches the next opcode dword, sign extending it into a quad word.
1364	*
1365	* @returns Strict VBox status code.
1366	* @param pIemCpu The IEM state.
1367	* @param pu64 Where to return the opcode quad word.
1368	*/
1369	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1370	{
1371	uint8_t const offOpcode = pIemCpu->offOpcode;
1372	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1373	return iemOpcodeGetNextS32SxU64Slow(pIemCpu, pu64);
1374
1375	int32_t i32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1376	pIemCpu->abOpcode[offOpcode + 1],
1377	pIemCpu->abOpcode[offOpcode + 2],
1378	pIemCpu->abOpcode[offOpcode + 3]);
1379	*pu64 = i32;
1380	pIemCpu->offOpcode = offOpcode + 4;
1381	return VINF_SUCCESS;
1382	}
1383
1384
1385	/**
1386	* Fetches the next opcode double word and sign extends it to a quad word,
1387	* returns automatically on failure.
1388	*
1389	* @param a_pu64 Where to return the opcode quad word.
1390	* @remark Implicitly references pIemCpu.
1391	*/
1392	#define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1393	do \
1394	{ \
1395	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pIemCpu, (a_pu64)); \
1396	if (rcStrict2 != VINF_SUCCESS) \
1397	return rcStrict2; \
1398	} while (0)
1399
1400
1401	/**
1402	* Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
1403	*
1404	* @returns Strict VBox status code.
1405	* @param pIemCpu The IEM state.
1406	* @param pu64 Where to return the opcode qword.
1407	*/
1408	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1409	{
1410	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 8);
1411	if (rcStrict == VINF_SUCCESS)
1412	{
1413	uint8_t offOpcode = pIemCpu->offOpcode;
1414	*pu64 = RT_MAKE_U64_FROM_U8(pIemCpu->abOpcode[offOpcode],
1415	pIemCpu->abOpcode[offOpcode + 1],
1416	pIemCpu->abOpcode[offOpcode + 2],
1417	pIemCpu->abOpcode[offOpcode + 3],
1418	pIemCpu->abOpcode[offOpcode + 4],
1419	pIemCpu->abOpcode[offOpcode + 5],
1420	pIemCpu->abOpcode[offOpcode + 6],
1421	pIemCpu->abOpcode[offOpcode + 7]);
1422	pIemCpu->offOpcode = offOpcode + 8;
1423	}
1424	else
1425	*pu64 = 0;
1426	return rcStrict;
1427	}
1428
1429
1430	/**
1431	* Fetches the next opcode qword.
1432	*
1433	* @returns Strict VBox status code.
1434	* @param pIemCpu The IEM state.
1435	* @param pu64 Where to return the opcode qword.
1436	*/
1437	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PIEMCPU pIemCpu, uint64_t *pu64)
1438	{
1439	uint8_t const offOpcode = pIemCpu->offOpcode;
1440	if (RT_UNLIKELY(offOpcode + 8 > pIemCpu->cbOpcode))
1441	return iemOpcodeGetNextU64Slow(pIemCpu, pu64);
1442
1443	*pu64 = RT_MAKE_U64_FROM_U8(pIemCpu->abOpcode[offOpcode],
1444	pIemCpu->abOpcode[offOpcode + 1],
1445	pIemCpu->abOpcode[offOpcode + 2],
1446	pIemCpu->abOpcode[offOpcode + 3],
1447	pIemCpu->abOpcode[offOpcode + 4],
1448	pIemCpu->abOpcode[offOpcode + 5],
1449	pIemCpu->abOpcode[offOpcode + 6],
1450	pIemCpu->abOpcode[offOpcode + 7]);
1451	pIemCpu->offOpcode = offOpcode + 8;
1452	return VINF_SUCCESS;
1453	}
1454
1455
1456	/**
1457	* Fetches the next opcode quad word, returns automatically on failure.
1458	*
1459	* @param a_pu64 Where to return the opcode quad word.
1460	* @remark Implicitly references pIemCpu.
1461	*/
1462	#define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1463	do \
1464	{ \
1465	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pIemCpu, (a_pu64)); \
1466	if (rcStrict2 != VINF_SUCCESS) \
1467	return rcStrict2; \
1468	} while (0)
1469
1470
1471	/** @name Misc Worker Functions.
1472	* @{
1473	*/
1474
1475
1476	/**
1477	* Validates a new SS segment.
1478	*
1479	* @returns VBox strict status code.
1480	* @param pIemCpu The IEM per CPU instance data.
1481	* @param pCtx The CPU context.
1482	* @param NewSS The new SS selctor.
1483	* @param uCpl The CPL to load the stack for.
1484	* @param pDesc Where to return the descriptor.
1485	*/
1486	static VBOXSTRICTRC iemMiscValidateNewSS(PIEMCPU pIemCpu, PCCPUMCTX pCtx, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc)
1487	{
1488	NOREF(pCtx);
1489
1490	/* Null selectors are not allowed (we're not called for dispatching
1491	interrupts with SS=0 in long mode). */
1492	if (!(NewSS & (X86_SEL_MASK \| X86_SEL_LDT)))
1493	{
1494	Log(("iemMiscValidateNewSSandRsp: #x - null selector -> #GP(0)\n", NewSS));
1495	return iemRaiseGeneralProtectionFault0(pIemCpu);
1496	}
1497
1498	/*
1499	* Read the descriptor.
1500	*/
1501	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pIemCpu, pDesc, NewSS);
1502	if (rcStrict != VINF_SUCCESS)
1503	return rcStrict;
1504
1505	/*
1506	* Perform the descriptor validation documented for LSS, POP SS and MOV SS.
1507	*/
1508	if (!pDesc->Legacy.Gen.u1DescType)
1509	{
1510	Log(("iemMiscValidateNewSSandRsp: %#x - system selector -> #GP\n", NewSS, pDesc->Legacy.Gen.u4Type));
1511	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1512	}
1513
1514	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1515	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1516	{
1517	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #GP\n", NewSS, pDesc->Legacy.Gen.u4Type));
1518	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1519	}
1520	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1521	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1522	{
1523	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #GP\n", NewSS, pDesc->Legacy.Gen.u4Type));
1524	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1525	}
1526	/** @todo testcase: check if the TSS.ssX RPL is checked. */
1527	if ((NewSS & X86_SEL_RPL) != uCpl)
1528	{
1529	Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #GP\n", NewSS, uCpl));
1530	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1531	}
1532	if (pDesc->Legacy.Gen.u2Dpl != uCpl)
1533	{
1534	Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #GP\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
1535	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1536	}
1537
1538	/* Is it there? */
1539	/** @todo testcase: Is this checked before the canonical / limit check below? */
1540	if (!pDesc->Legacy.Gen.u1Present)
1541	{
1542	Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
1543	return iemRaiseSelectorNotPresentBySelector(pIemCpu, NewSS);
1544	}
1545
1546	return VINF_SUCCESS;
1547	}
1548
1549
1550	/** @} */
1551
1552	/** @name Raising Exceptions.
1553	*
1554	* @{
1555	*/
1556
1557	/** @name IEM_XCPT_FLAGS_XXX - flags for iemRaiseXcptOrInt.
1558	* @{ */
1559	/** CPU exception. */
1560	#define IEM_XCPT_FLAGS_T_CPU_XCPT RT_BIT_32(0)
1561	/** External interrupt (from PIC, APIC, whatever). */
1562	#define IEM_XCPT_FLAGS_T_EXT_INT RT_BIT_32(1)
1563	/** Software interrupt (int, into or bound). */
1564	#define IEM_XCPT_FLAGS_T_SOFT_INT RT_BIT_32(2)
1565	/** Takes an error code. */
1566	#define IEM_XCPT_FLAGS_ERR RT_BIT_32(3)
1567	/** Takes a CR2. */
1568	#define IEM_XCPT_FLAGS_CR2 RT_BIT_32(4)
1569	/** Generated by the breakpoint instruction. */
1570	#define IEM_XCPT_FLAGS_BP_INSTR RT_BIT_32(5)
1571	/** @} */
1572
1573	/**
1574	* Loads the specified stack far pointer from the TSS.
1575	*
1576	* @returns VBox strict status code.
1577	* @param pIemCpu The IEM per CPU instance data.
1578	* @param pCtx The CPU context.
1579	* @param uCpl The CPL to load the stack for.
1580	* @param pSelSS Where to return the new stack segment.
1581	* @param puEsp Where to return the new stack pointer.
1582	*/
1583	static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PIEMCPU pIemCpu, PCCPUMCTX pCtx, uint8_t uCpl,
1584	PRTSEL pSelSS, uint32_t *puEsp)
1585	{
1586	VBOXSTRICTRC rcStrict;
1587	Assert(uCpl < 4);
1588	puEsp = 0; / make gcc happy */
1589	pSelSS = 0; / make gcc happy */
1590
1591	switch (pCtx->trHid.Attr.n.u4Type)
1592	{
1593	/*
1594	* 16-bit TSS (X86TSS16).
1595	*/
1596	case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed();
1597	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1598	{
1599	uint32_t off = uCpl * 4 + 2;
1600	if (off + 4 > pCtx->trHid.u32Limit)
1601	{
1602	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pCtx->trHid.u32Limit));
1603	return iemRaiseTaskSwitchFaultCurrentTSS(pIemCpu);
1604	}
1605
1606	uint32_t u32Tmp = 0; /* gcc maybe... */
1607	rcStrict = iemMemFetchSysU32(pIemCpu, &u32Tmp, UINT8_MAX, pCtx->trHid.u64Base + off);
1608	if (rcStrict == VINF_SUCCESS)
1609	{
1610	*puEsp = RT_LOWORD(u32Tmp);
1611	*pSelSS = RT_HIWORD(u32Tmp);
1612	return VINF_SUCCESS;
1613	}
1614	break;
1615	}
1616
1617	/*
1618	* 32-bit TSS (X86TSS32).
1619	*/
1620	case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed();
1621	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1622	{
1623	uint32_t off = uCpl * 8 + 4;
1624	if (off + 7 > pCtx->trHid.u32Limit)
1625	{
1626	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pCtx->trHid.u32Limit));
1627	return iemRaiseTaskSwitchFaultCurrentTSS(pIemCpu);
1628	}
1629
1630	uint64_t u64Tmp;
1631	rcStrict = iemMemFetchSysU64(pIemCpu, &u64Tmp, UINT8_MAX, pCtx->trHid.u64Base + off);
1632	if (rcStrict == VINF_SUCCESS)
1633	{
1634	*puEsp = u64Tmp & UINT32_MAX;
1635	*pSelSS = (RTSEL)(u64Tmp >> 32);
1636	return VINF_SUCCESS;
1637	}
1638	break;
1639	}
1640
1641	default:
1642	AssertFailedReturn(VERR_INTERNAL_ERROR_2);
1643	}
1644	return rcStrict;
1645	}
1646
1647
1648	/**
1649	* Adjust the CPU state according to the exception being raised.
1650	*
1651	* @param pCtx The CPU context.
1652	* @param u8Vector The exception that has been raised.
1653	*/
1654	DECLINLINE(void) iemRaiseXcptAdjustState(PCPUMCTX pCtx, uint8_t u8Vector)
1655	{
1656	switch (u8Vector)
1657	{
1658	case X86_XCPT_DB:
1659	pCtx->dr[7] &= ~X86_DR7_GD;
1660	break;
1661	/** @todo Read the AMD and Intel exception reference... */
1662	}
1663	}
1664
1665
1666	/**
1667	* Implements exceptions and interrupts for real mode.
1668	*
1669	* @returns VBox strict status code.
1670	* @param pIemCpu The IEM per CPU instance data.
1671	* @param pCtx The CPU context.
1672	* @param cbInstr The number of bytes to offset rIP by in the return
1673	* address.
1674	* @param u8Vector The interrupt / exception vector number.
1675	* @param fFlags The flags.
1676	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
1677	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
1678	*/
1679	static VBOXSTRICTRC
1680	iemRaiseXcptOrIntInRealMode(PIEMCPU pIemCpu,
1681	PCPUMCTX pCtx,
1682	uint8_t cbInstr,
1683	uint8_t u8Vector,
1684	uint32_t fFlags,
1685	uint16_t uErr,
1686	uint64_t uCr2)
1687	{
1688	AssertReturn(pIemCpu->enmCpuMode == IEMMODE_16BIT, VERR_INTERNAL_ERROR_3);
1689	NOREF(uErr); NOREF(uCr2);
1690
1691	/*
1692	* Read the IDT entry.
1693	*/
1694	if (pCtx->idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
1695	{
1696	Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pCtx->idtr.cbIdt));
1697	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1698	}
1699	RTFAR16 Idte;
1700	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pIemCpu, (uint32_t *)&Idte, UINT8_MAX,
1701	pCtx->idtr.pIdt + UINT32_C(4) * u8Vector);
1702	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1703	return rcStrict;
1704
1705	/*
1706	* Push the stack frame.
1707	*/
1708	uint16_t *pu16Frame;
1709	uint64_t uNewRsp;
1710	rcStrict = iemMemStackPushBeginSpecial(pIemCpu, 6, (void **)&pu16Frame, &uNewRsp);
1711	if (rcStrict != VINF_SUCCESS)
1712	return rcStrict;
1713
1714	pu16Frame[2] = (uint16_t)pCtx->eflags.u;
1715	pu16Frame[1] = (uint16_t)pCtx->cs;
1716	pu16Frame[0] = pCtx->ip + cbInstr;
1717	rcStrict = iemMemStackPushCommitSpecial(pIemCpu, pu16Frame, uNewRsp);
1718	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1719	return rcStrict;
1720
1721	/*
1722	* Load the vector address into cs:ip and make exception specific state
1723	* adjustments.
1724	*/
1725	pCtx->cs = Idte.sel;
1726	pCtx->csHid.u64Base = (uint32_t)Idte.sel << 4;
1727	/** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
1728	pCtx->rip = Idte.off;
1729	pCtx->eflags.Bits.u1IF = 0;
1730
1731	/** @todo do we actually do this in real mode? */
1732	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1733	iemRaiseXcptAdjustState(pCtx, u8Vector);
1734
1735	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
1736	}
1737
1738
1739	/**
1740	* Implements exceptions and interrupts for protected mode.
1741	*
1742	* @returns VBox strict status code.
1743	* @param pIemCpu The IEM per CPU instance data.
1744	* @param pCtx The CPU context.
1745	* @param cbInstr The number of bytes to offset rIP by in the return
1746	* address.
1747	* @param u8Vector The interrupt / exception vector number.
1748	* @param fFlags The flags.
1749	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
1750	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
1751	*/
1752	static VBOXSTRICTRC
1753	iemRaiseXcptOrIntInProtMode(PIEMCPU pIemCpu,
1754	PCPUMCTX pCtx,
1755	uint8_t cbInstr,
1756	uint8_t u8Vector,
1757	uint32_t fFlags,
1758	uint16_t uErr,
1759	uint64_t uCr2)
1760	{
1761	NOREF(cbInstr);
1762
1763	/*
1764	* Read the IDT entry.
1765	*/
1766	if (pCtx->idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
1767	{
1768	Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pCtx->idtr.cbIdt));
1769	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1770	}
1771	X86DESC Idte;
1772	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pIemCpu, &Idte.u, UINT8_MAX,
1773	pCtx->idtr.pIdt + UINT32_C(8) * u8Vector);
1774	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1775	return rcStrict;
1776	LogFlow(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
1777	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
1778	Idte.Gate.u4ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
1779
1780	/*
1781	* Check the descriptor type, DPL and such.
1782	* ASSUMES this is done in the same order as described for call-gate calls.
1783	*/
1784	if (Idte.Gate.u1DescType)
1785	{
1786	Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
1787	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1788	}
1789	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
1790	switch (Idte.Gate.u4Type)
1791	{
1792	case X86_SEL_TYPE_SYS_UNDEFINED:
1793	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
1794	case X86_SEL_TYPE_SYS_LDT:
1795	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1796	case X86_SEL_TYPE_SYS_286_CALL_GATE:
1797	case X86_SEL_TYPE_SYS_UNDEFINED2:
1798	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
1799	case X86_SEL_TYPE_SYS_UNDEFINED3:
1800	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1801	case X86_SEL_TYPE_SYS_386_CALL_GATE:
1802	case X86_SEL_TYPE_SYS_UNDEFINED4:
1803	{
1804	/** @todo check what actually happens when the type is wrong...
1805	* esp. call gates. */
1806	Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
1807	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1808	}
1809
1810	case X86_SEL_TYPE_SYS_286_INT_GATE:
1811	case X86_SEL_TYPE_SYS_386_INT_GATE:
1812	fEflToClear \|= X86_EFL_IF;
1813	break;
1814
1815	case X86_SEL_TYPE_SYS_TASK_GATE:
1816	/** @todo task gates. */
1817	AssertFailedReturn(VERR_NOT_SUPPORTED);
1818
1819	case X86_SEL_TYPE_SYS_286_TRAP_GATE:
1820	case X86_SEL_TYPE_SYS_386_TRAP_GATE:
1821	break;
1822
1823	IEM_NOT_REACHED_DEFAULT_CASE_RET();
1824	}
1825
1826	/* Check DPL against CPL if applicable. */
1827	if (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
1828	{
1829	if (pIemCpu->uCpl > Idte.Gate.u2Dpl)
1830	{
1831	Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pIemCpu->uCpl, Idte.Gate.u2Dpl));
1832	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1833	}
1834	}
1835
1836	/* Is it there? */
1837	if (!Idte.Gate.u1Present)
1838	{
1839	Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
1840	return iemRaiseSelectorNotPresentWithErr(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1841	}
1842
1843	/* A null CS is bad. */
1844	RTSEL NewCS = Idte.Gate.u16Sel;
1845	if (!(NewCS & (X86_SEL_MASK \| X86_SEL_LDT)))
1846	{
1847	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
1848	return iemRaiseGeneralProtectionFault0(pIemCpu);
1849	}
1850
1851	/* Fetch the descriptor for the new CS. */
1852	IEMSELDESC DescCS;
1853	rcStrict = iemMemFetchSelDesc(pIemCpu, &DescCS, NewCS);
1854	if (rcStrict != VINF_SUCCESS)
1855	{
1856	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
1857	return rcStrict;
1858	}
1859
1860	/* Must be a code segment. */
1861	if (!DescCS.Legacy.Gen.u1DescType)
1862	{
1863	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
1864	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1865	}
1866	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
1867	{
1868	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
1869	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1870	}
1871
1872	/* Don't allow lowering the privilege level. */
1873	/** @todo Does the lowering of privileges apply to software interrupts
1874	* only? This has bearings on the more-privileged or
1875	* same-privilege stack behavior further down. A testcase would
1876	* be nice. */
1877	if (DescCS.Legacy.Gen.u2Dpl > pIemCpu->uCpl)
1878	{
1879	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
1880	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pIemCpu->uCpl));
1881	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1882	}
1883	/** @todo is the RPL of the interrupt/trap gate descriptor checked? */
1884
1885	/* Check the new EIP against the new CS limit. */
1886	uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
1887	\|\| Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
1888	? Idte.Gate.u16OffsetLow
1889	: Idte.Gate.u16OffsetLow \| ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
1890	uint32_t cbLimitCS = X86DESC_LIMIT(DescCS.Legacy);
1891	if (DescCS.Legacy.Gen.u1Granularity)
1892	cbLimitCS = (cbLimitCS << PAGE_SHIFT) \| PAGE_OFFSET_MASK;
1893	if (uNewEip > cbLimitCS)
1894	{
1895	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
1896	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pIemCpu->uCpl));
1897	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1898	}
1899
1900	/* Make sure the selector is present. */
1901	if (!DescCS.Legacy.Gen.u1Present)
1902	{
1903	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
1904	return iemRaiseSelectorNotPresentBySelector(pIemCpu, NewCS);
1905	}
1906
1907	/*
1908	* If the privilege level changes, we need to get a new stack from the TSS.
1909	* This in turns means validating the new SS and ESP...
1910	*/
1911	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
1912	? pIemCpu->uCpl : DescCS.Legacy.Gen.u2Dpl;
1913	if (uNewCpl != pIemCpu->uCpl)
1914	{
1915	RTSEL NewSS;
1916	uint32_t uNewEsp;
1917	rcStrict = iemRaiseLoadStackFromTss32Or16(pIemCpu, pCtx, uNewCpl, &NewSS, &uNewEsp);
1918	if (rcStrict != VINF_SUCCESS)
1919	return rcStrict;
1920
1921	IEMSELDESC DescSS;
1922	rcStrict = iemMiscValidateNewSS(pIemCpu, pCtx, NewSS, uNewCpl, &DescSS);
1923	if (rcStrict != VINF_SUCCESS)
1924	return rcStrict;
1925
1926	/* Check that there is sufficient space for the stack frame. */
1927	uint32_t cbLimitSS = X86DESC_LIMIT(DescSS.Legacy);
1928	if (DescSS.Legacy.Gen.u1Granularity)
1929	cbLimitSS = (cbLimitSS << PAGE_SHIFT) \| PAGE_OFFSET_MASK;
1930	AssertReturn(!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
1931
1932	uint8_t const cbStackFrame = fFlags & IEM_XCPT_FLAGS_ERR ? 24 : 20;
1933	if ( uNewEsp - 1 > cbLimitSS
1934	\|\| uNewEsp < cbStackFrame)
1935	{
1936	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
1937	u8Vector, NewSS, uNewEsp, cbStackFrame));
1938	return iemRaiseSelectorBoundsBySelector(pIemCpu, NewSS);
1939	}
1940
1941	/*
1942	* Start making changes.
1943	*/
1944
1945	/* Create the stack frame. */
1946	RTPTRUNION uStackFrame;
1947	rcStrict = iemMemMap(pIemCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
1948	uNewEsp - cbStackFrame + X86DESC_BASE(DescSS.Legacy), IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
1949	if (rcStrict != VINF_SUCCESS)
1950	return rcStrict;
1951	void * const pvStackFrame = uStackFrame.pv;
1952
1953	if (fFlags & IEM_XCPT_FLAGS_ERR)
1954	*uStackFrame.pu32++ = uErr;
1955	uStackFrame.pu32[0] = (fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
1956	? pCtx->eip + cbInstr : pCtx->eip;
1957	uStackFrame.pu32[1] = (pCtx->cs & ~X86_SEL_RPL) \| pIemCpu->uCpl;
1958	uStackFrame.pu32[2] = pCtx->eflags.u;
1959	uStackFrame.pu32[3] = pCtx->esp;
1960	uStackFrame.pu32[4] = pCtx->ss;
1961	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
1962	if (rcStrict != VINF_SUCCESS)
1963	return rcStrict;
1964
1965	/* Mark the selectors 'accessed' (hope this is the correct time). */
1966	/** @todo testcase: excatly _when_ are the accessed bits set - before or
1967	* after pushing the stack frame? (Write protect the gdt + stack to
1968	* find out.) */
1969	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1970	{
1971	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewCS);
1972	if (rcStrict != VINF_SUCCESS)
1973	return rcStrict;
1974	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1975	}
1976
1977	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1978	{
1979	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewSS);
1980	if (rcStrict != VINF_SUCCESS)
1981	return rcStrict;
1982	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1983	}
1984
1985	/*
1986	* Start commint the register changes (joins with the DPL=CPL branch).
1987	*/
1988	pCtx->ss = NewSS;
1989	pCtx->ssHid.u32Limit = cbLimitSS;
1990	pCtx->ssHid.u64Base = X86DESC_BASE(DescSS.Legacy);
1991	pCtx->ssHid.Attr.u = X86DESC_GET_HID_ATTR(DescSS.Legacy);
1992	pCtx->rsp = uNewEsp - cbStackFrame; /** @todo Is the high word cleared for 16-bit stacks and/or interrupt handlers? */
1993	pIemCpu->uCpl = uNewCpl;
1994	}
1995	/*
1996	* Same privilege, no stack change and smaller stack frame.
1997	*/
1998	else
1999	{
2000	uint64_t uNewRsp;
2001	RTPTRUNION uStackFrame;
2002	uint8_t const cbStackFrame = fFlags & IEM_XCPT_FLAGS_ERR ? 16 : 12;
2003	rcStrict = iemMemStackPushBeginSpecial(pIemCpu, cbStackFrame, &uStackFrame.pv, &uNewRsp);
2004	if (rcStrict != VINF_SUCCESS)
2005	return rcStrict;
2006	void * const pvStackFrame = uStackFrame.pv;
2007
2008	if (fFlags & IEM_XCPT_FLAGS_ERR)
2009	*uStackFrame.pu32++ = uErr;
2010	uStackFrame.pu32[0] = (fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
2011	? pCtx->eip + cbInstr : pCtx->eip;
2012	uStackFrame.pu32[1] = (pCtx->cs & ~X86_SEL_RPL) \| pIemCpu->uCpl;
2013	uStackFrame.pu32[2] = pCtx->eflags.u;
2014	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
2015	if (rcStrict != VINF_SUCCESS)
2016	return rcStrict;
2017
2018	/* Mark the CS selector as 'accessed'. */
2019	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2020	{
2021	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewCS);
2022	if (rcStrict != VINF_SUCCESS)
2023	return rcStrict;
2024	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2025	}
2026
2027	/*
2028	* Start committing the register changes (joins with the other branch).
2029	*/
2030	pCtx->rsp = uNewRsp;
2031	}
2032
2033	/* ... register committing continues. */
2034	pCtx->cs = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
2035	pCtx->csHid.u32Limit = cbLimitCS;
2036	pCtx->csHid.u64Base = X86DESC_BASE(DescCS.Legacy);
2037	pCtx->csHid.Attr.u = X86DESC_GET_HID_ATTR(DescCS.Legacy);
2038
2039	pCtx->rip = uNewEip;
2040	pCtx->rflags.u &= ~fEflToClear;
2041
2042	if (fFlags & IEM_XCPT_FLAGS_CR2)
2043	pCtx->cr2 = uCr2;
2044
2045	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
2046	iemRaiseXcptAdjustState(pCtx, u8Vector);
2047
2048	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
2049	}
2050
2051
2052	/**
2053	* Implements exceptions and interrupts for V8086 mode.
2054	*
2055	* @returns VBox strict status code.
2056	* @param pIemCpu The IEM per CPU instance data.
2057	* @param pCtx The CPU context.
2058	* @param cbInstr The number of bytes to offset rIP by in the return
2059	* address.
2060	* @param u8Vector The interrupt / exception vector number.
2061	* @param fFlags The flags.
2062	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2063	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2064	*/
2065	static VBOXSTRICTRC
2066	iemRaiseXcptOrIntInV8086Mode(PIEMCPU pIemCpu,
2067	PCPUMCTX pCtx,
2068	uint8_t cbInstr,
2069	uint8_t u8Vector,
2070	uint32_t fFlags,
2071	uint16_t uErr,
2072	uint64_t uCr2)
2073	{
2074	NOREF(pIemCpu); NOREF(pCtx); NOREF(cbInstr); NOREF(u8Vector); NOREF(fFlags); NOREF(uErr); NOREF(uCr2);
2075	AssertMsgFailed(("V8086 exception / interrupt dispatching\n"));
2076	return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
2077	}
2078
2079
2080	/**
2081	* Implements exceptions and interrupts for long mode.
2082	*
2083	* @returns VBox strict status code.
2084	* @param pIemCpu The IEM per CPU instance data.
2085	* @param pCtx The CPU context.
2086	* @param cbInstr The number of bytes to offset rIP by in the return
2087	* address.
2088	* @param u8Vector The interrupt / exception vector number.
2089	* @param fFlags The flags.
2090	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2091	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2092	*/
2093	static VBOXSTRICTRC
2094	iemRaiseXcptOrIntInLongMode(PIEMCPU pIemCpu,
2095	PCPUMCTX pCtx,
2096	uint8_t cbInstr,
2097	uint8_t u8Vector,
2098	uint32_t fFlags,
2099	uint16_t uErr,
2100	uint64_t uCr2)
2101	{
2102	NOREF(pIemCpu); NOREF(pCtx); NOREF(cbInstr); NOREF(u8Vector); NOREF(fFlags); NOREF(uErr); NOREF(uCr2);
2103	AssertMsgFailed(("long mode exception / interrupt dispatching\n"));
2104	return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
2105	}
2106
2107
2108	/**
2109	* Implements exceptions and interrupts.
2110	*
2111	* All exceptions and interrupts goes thru this function!
2112	*
2113	* @returns VBox strict status code.
2114	* @param pIemCpu The IEM per CPU instance data.
2115	* @param cbInstr The number of bytes to offset rIP by in the return
2116	* address.
2117	* @param u8Vector The interrupt / exception vector number.
2118	* @param fFlags The flags.
2119	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2120	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2121	*/
2122	DECL_NO_INLINE(static, VBOXSTRICTRC)
2123	iemRaiseXcptOrInt(PIEMCPU pIemCpu,
2124	uint8_t cbInstr,
2125	uint8_t u8Vector,
2126	uint32_t fFlags,
2127	uint16_t uErr,
2128	uint64_t uCr2)
2129	{
2130	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2131
2132	/*
2133	* Do recursion accounting.
2134	*/
2135	uint8_t const uPrevXcpt = pIemCpu->uCurXcpt;
2136	uint32_t const fPrevXcpt = pIemCpu->fCurXcpt;
2137	if (pIemCpu->cXcptRecursions == 0)
2138	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
2139	u8Vector, pCtx->cs, pCtx->rip, cbInstr, fFlags, uErr, uCr2));
2140	else
2141	{
2142	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
2143	u8Vector, pCtx->cs, pCtx->rip, cbInstr, fFlags, uErr, uCr2, pIemCpu->uCurXcpt, pIemCpu->cXcptRecursions + 1, fPrevXcpt));
2144
2145	/** @todo double and tripple faults. */
2146	AssertReturn(pIemCpu->cXcptRecursions < 3, VERR_IEM_ASPECT_NOT_IMPLEMENTED);
2147
2148	/** @todo set X86_TRAP_ERR_EXTERNAL when appropriate.
2149	if (fPrevXcpt & IEM_XCPT_FLAGS_T_EXT_INT)
2150	{
2151	....
2152	} */
2153	}
2154	pIemCpu->cXcptRecursions++;
2155	pIemCpu->uCurXcpt = u8Vector;
2156	pIemCpu->fCurXcpt = fFlags;
2157
2158	/*
2159	* Extensive logging.
2160	*/
2161	#ifdef LOG_ENABLED
2162	if (LogIs3Enabled())
2163	{
2164	PVM pVM = IEMCPU_TO_VM(pIemCpu);
2165	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
2166	char szRegs[4096];
2167	DBGFR3RegPrintf(pVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
2168	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
2169	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
2170	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
2171	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
2172	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
2173	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
2174	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
2175	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
2176	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
2177	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
2178	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
2179	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
2180	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
2181	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
2182	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
2183	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
2184	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
2185	" efer=%016VR{efer}\n"
2186	" pat=%016VR{pat}\n"
2187	" sf_mask=%016VR{sf_mask}\n"
2188	"krnl_gs_base=%016VR{krnl_gs_base}\n"
2189	" lstar=%016VR{lstar}\n"
2190	" star=%016VR{star} cstar=%016VR{cstar}\n"
2191	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
2192	);
2193
2194	char szInstr[256];
2195	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0,
2196	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
2197	szInstr, sizeof(szInstr), NULL);
2198	Log3(("%s%s\n", szRegs, szInstr));
2199	}
2200	#endif /* LOG_ENABLED */
2201
2202	/*
2203	* Call the mode specific worker function.
2204	*/
2205	VBOXSTRICTRC rcStrict;
2206	if (!(pCtx->cr0 & X86_CR0_PE))
2207	rcStrict = iemRaiseXcptOrIntInRealMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2208	else if (pCtx->msrEFER & MSR_K6_EFER_LMA)
2209	rcStrict = iemRaiseXcptOrIntInLongMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2210	else if (!pCtx->eflags.Bits.u1VM)
2211	rcStrict = iemRaiseXcptOrIntInProtMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2212	else
2213	rcStrict = iemRaiseXcptOrIntInV8086Mode(pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2214
2215	/*
2216	* Unwind.
2217	*/
2218	pIemCpu->cXcptRecursions--;
2219	pIemCpu->uCurXcpt = uPrevXcpt;
2220	pIemCpu->fCurXcpt = fPrevXcpt;
2221	LogFlow(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv\n",
2222	VBOXSTRICTRC_VAL(rcStrict), u8Vector, pCtx->cs, pCtx->rip, pCtx->ss, pCtx->esp));
2223	return rcStrict;
2224	}
2225
2226
2227	/** \#DE - 00. */
2228	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDivideError(PIEMCPU pIemCpu)
2229	{
2230	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2231	}
2232
2233
2234	/** \#DB - 01. */
2235	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDebugException(PIEMCPU pIemCpu)
2236	{
2237	/** @todo set/clear RF. */
2238	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2239	}
2240
2241
2242	/** \#UD - 06. */
2243	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseUndefinedOpcode(PIEMCPU pIemCpu)
2244	{
2245	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2246	}
2247
2248
2249	/** \#NM - 07. */
2250	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDeviceNotAvailable(PIEMCPU pIemCpu)
2251	{
2252	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2253	}
2254
2255
2256	#ifdef SOME_UNUSED_FUNCTION
2257	/** \#TS(err) - 0a. */
2258	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseTaskSwitchFaultWithErr(PIEMCPU pIemCpu, uint16_t uErr)
2259	{
2260	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
2261	}
2262	#endif
2263
2264
2265	/** \#TS(tr) - 0a. */
2266	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseTaskSwitchFaultCurrentTSS(PIEMCPU pIemCpu)
2267	{
2268	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2269	pIemCpu->CTX_SUFF(pCtx)->tr, 0);
2270	}
2271
2272
2273	/** \#NP(err) - 0b. */
2274	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentWithErr(PIEMCPU pIemCpu, uint16_t uErr)
2275	{
2276	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
2277	}
2278
2279
2280	/** \#NP(seg) - 0b. */
2281	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySegReg(PIEMCPU pIemCpu, uint32_t iSegReg)
2282	{
2283	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2284	iemSRegFetchU16(pIemCpu, iSegReg) & ~X86_SEL_RPL, 0);
2285	}
2286
2287
2288	/** \#NP(sel) - 0b. */
2289	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel)
2290	{
2291	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2292	uSel & ~X86_SEL_RPL, 0);
2293	}
2294
2295
2296	/** \#SS(seg) - 0c. */
2297	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel)
2298	{
2299	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2300	uSel & ~X86_SEL_RPL, 0);
2301	}
2302
2303
2304	/** \#GP(n) - 0d. */
2305	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFault(PIEMCPU pIemCpu, uint16_t uErr)
2306	{
2307	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
2308	}
2309
2310
2311	/** \#GP(0) - 0d. */
2312	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFault0(PIEMCPU pIemCpu)
2313	{
2314	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2315	}
2316
2317
2318	/** \#GP(sel) - 0d. */
2319	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFaultBySelector(PIEMCPU pIemCpu, RTSEL Sel)
2320	{
2321	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2322	Sel & ~X86_SEL_RPL, 0);
2323	}
2324
2325
2326	/** \#GP(0) - 0d. */
2327	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseNotCanonical(PIEMCPU pIemCpu)
2328	{
2329	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2330	}
2331
2332
2333	/** \#GP(sel) - 0d. */
2334	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorBounds(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess)
2335	{
2336	NOREF(iSegReg); NOREF(fAccess);
2337	return iemRaiseXcptOrInt(pIemCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
2338	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2339	}
2340
2341
2342	/** \#GP(sel) - 0d. */
2343	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorBoundsBySelector(PIEMCPU pIemCpu, RTSEL Sel)
2344	{
2345	NOREF(Sel);
2346	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2347	}
2348
2349
2350	/** \#GP(sel) - 0d. */
2351	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorInvalidAccess(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess)
2352	{
2353	NOREF(iSegReg); NOREF(fAccess);
2354	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2355	}
2356
2357
2358	/** \#PF(n) - 0e. */
2359	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaisePageFault(PIEMCPU pIemCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
2360	{
2361	uint16_t uErr;
2362	switch (rc)
2363	{
2364	case VERR_PAGE_NOT_PRESENT:
2365	case VERR_PAGE_TABLE_NOT_PRESENT:
2366	case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
2367	case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
2368	uErr = 0;
2369	break;
2370
2371	default:
2372	AssertMsgFailed(("%Rrc\n", rc));
2373	case VERR_ACCESS_DENIED:
2374	uErr = X86_TRAP_PF_P;
2375	break;
2376
2377	/** @todo reserved */
2378	}
2379
2380	if (pIemCpu->uCpl == 3)
2381	uErr \|= X86_TRAP_PF_US;
2382
2383	if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
2384	&& ( (pIemCpu->CTX_SUFF(pCtx)->cr4 & X86_CR4_PAE)
2385	&& (pIemCpu->CTX_SUFF(pCtx)->msrEFER & MSR_K6_EFER_NXE) ) )
2386	uErr \|= X86_TRAP_PF_ID;
2387
2388	/* Note! RW access callers reporting a WRITE protection fault, will clear
2389	the READ flag before calling. So, read-modify-write accesses (RW)
2390	can safely be reported as READ faults. */
2391	if ((fAccess & (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
2392	uErr \|= X86_TRAP_PF_RW;
2393
2394	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR \| IEM_XCPT_FLAGS_CR2,
2395	uErr, GCPtrWhere);
2396	}
2397
2398
2399	/** \#MF(0) - 10. */
2400	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseMathFault(PIEMCPU pIemCpu)
2401	{
2402	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2403	}
2404
2405
2406	/** \#AC(0) - 11. */
2407	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseAlignmentCheckException(PIEMCPU pIemCpu)
2408	{
2409	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2410	}
2411
2412
2413	/**
2414	* Macro for calling iemCImplRaiseDivideError().
2415	*
2416	* This enables us to add/remove arguments and force different levels of
2417	* inlining as we wish.
2418	*
2419	* @return Strict VBox status code.
2420	*/
2421	#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
2422	IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
2423	{
2424	NOREF(cbInstr);
2425	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2426	}
2427
2428
2429	/**
2430	* Macro for calling iemCImplRaiseInvalidLockPrefix().
2431	*
2432	* This enables us to add/remove arguments and force different levels of
2433	* inlining as we wish.
2434	*
2435	* @return Strict VBox status code.
2436	*/
2437	#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
2438	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
2439	{
2440	NOREF(cbInstr);
2441	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2442	}
2443
2444
2445	/**
2446	* Macro for calling iemCImplRaiseInvalidOpcode().
2447	*
2448	* This enables us to add/remove arguments and force different levels of
2449	* inlining as we wish.
2450	*
2451	* @return Strict VBox status code.
2452	*/
2453	#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
2454	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
2455	{
2456	NOREF(cbInstr);
2457	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2458	}
2459
2460
2461	/** @} */
2462
2463
2464	/*
2465	*
2466	* Helpers routines.
2467	* Helpers routines.
2468	* Helpers routines.
2469	*
2470	*/
2471
2472	/**
2473	* Recalculates the effective operand size.
2474	*
2475	* @param pIemCpu The IEM state.
2476	*/
2477	static void iemRecalEffOpSize(PIEMCPU pIemCpu)
2478	{
2479	switch (pIemCpu->enmCpuMode)
2480	{
2481	case IEMMODE_16BIT:
2482	pIemCpu->enmEffOpSize = pIemCpu->fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
2483	break;
2484	case IEMMODE_32BIT:
2485	pIemCpu->enmEffOpSize = pIemCpu->fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
2486	break;
2487	case IEMMODE_64BIT:
2488	switch (pIemCpu->fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
2489	{
2490	case 0:
2491	pIemCpu->enmEffOpSize = pIemCpu->enmDefOpSize;
2492	break;
2493	case IEM_OP_PRF_SIZE_OP:
2494	pIemCpu->enmEffOpSize = IEMMODE_16BIT;
2495	break;
2496	case IEM_OP_PRF_SIZE_REX_W:
2497	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
2498	pIemCpu->enmEffOpSize = IEMMODE_64BIT;
2499	break;
2500	}
2501	break;
2502	default:
2503	AssertFailed();
2504	}
2505	}
2506
2507
2508	/**
2509	* Sets the default operand size to 64-bit and recalculates the effective
2510	* operand size.
2511	*
2512	* @param pIemCpu The IEM state.
2513	*/
2514	static void iemRecalEffOpSize64Default(PIEMCPU pIemCpu)
2515	{
2516	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2517	pIemCpu->enmDefOpSize = IEMMODE_64BIT;
2518	if ((pIemCpu->fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
2519	pIemCpu->enmEffOpSize = IEMMODE_64BIT;
2520	else
2521	pIemCpu->enmEffOpSize = IEMMODE_16BIT;
2522	}
2523
2524
2525	/*
2526	*
2527	* Common opcode decoders.
2528	* Common opcode decoders.
2529	* Common opcode decoders.
2530	*
2531	*/
2532	#include <iprt/mem.h>
2533
2534	/**
2535	* Used to add extra details about a stub case.
2536	* @param pIemCpu The IEM per CPU state.
2537	*/
2538	static void iemOpStubMsg2(PIEMCPU pIemCpu)
2539	{
2540	PVM pVM = IEMCPU_TO_VM(pIemCpu);
2541	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
2542	char szRegs[4096];
2543	DBGFR3RegPrintf(pVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
2544	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
2545	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
2546	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
2547	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
2548	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
2549	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
2550	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
2551	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
2552	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
2553	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
2554	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
2555	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
2556	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
2557	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
2558	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
2559	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
2560	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
2561	" efer=%016VR{efer}\n"
2562	" pat=%016VR{pat}\n"
2563	" sf_mask=%016VR{sf_mask}\n"
2564	"krnl_gs_base=%016VR{krnl_gs_base}\n"
2565	" lstar=%016VR{lstar}\n"
2566	" star=%016VR{star} cstar=%016VR{cstar}\n"
2567	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
2568	);
2569
2570	char szInstr[256];
2571	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0,
2572	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
2573	szInstr, sizeof(szInstr), NULL);
2574
2575	RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
2576	}
2577
2578
2579	/** Stubs an opcode. */
2580	#define FNIEMOP_STUB(a_Name) \
2581	FNIEMOP_DEF(a_Name) \
2582	{ \
2583	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
2584	iemOpStubMsg2(pIemCpu); \
2585	RTAssertPanic(); \
2586	return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
2587	} \
2588	typedef int ignore_semicolon
2589
2590	/** Stubs an opcode. */
2591	#define FNIEMOP_STUB_1(a_Name, a_Type0, a_Name0) \
2592	FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2593	{ \
2594	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
2595	iemOpStubMsg2(pIemCpu); \
2596	RTAssertPanic(); \
2597	NOREF(a_Name0); \
2598	return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
2599	} \
2600	typedef int ignore_semicolon
2601
2602
2603
2604	/** @name Register Access.
2605	* @{
2606	*/
2607
2608	/**
2609	* Gets a reference (pointer) to the specified hidden segment register.
2610	*
2611	* @returns Hidden register reference.
2612	* @param pIemCpu The per CPU data.
2613	* @param iSegReg The segment register.
2614	*/
2615	static PCPUMSELREGHID iemSRegGetHid(PIEMCPU pIemCpu, uint8_t iSegReg)
2616	{
2617	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2618	switch (iSegReg)
2619	{
2620	case X86_SREG_ES: return &pCtx->esHid;
2621	case X86_SREG_CS: return &pCtx->csHid;
2622	case X86_SREG_SS: return &pCtx->ssHid;
2623	case X86_SREG_DS: return &pCtx->dsHid;
2624	case X86_SREG_FS: return &pCtx->fsHid;
2625	case X86_SREG_GS: return &pCtx->gsHid;
2626	}
2627	AssertFailedReturn(NULL);
2628	}
2629
2630
2631	/**
2632	* Gets a reference (pointer) to the specified segment register (the selector
2633	* value).
2634	*
2635	* @returns Pointer to the selector variable.
2636	* @param pIemCpu The per CPU data.
2637	* @param iSegReg The segment register.
2638	*/
2639	static uint16_t *iemSRegRef(PIEMCPU pIemCpu, uint8_t iSegReg)
2640	{
2641	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2642	switch (iSegReg)
2643	{
2644	case X86_SREG_ES: return &pCtx->es;
2645	case X86_SREG_CS: return &pCtx->cs;
2646	case X86_SREG_SS: return &pCtx->ss;
2647	case X86_SREG_DS: return &pCtx->ds;
2648	case X86_SREG_FS: return &pCtx->fs;
2649	case X86_SREG_GS: return &pCtx->gs;
2650	}
2651	AssertFailedReturn(NULL);
2652	}
2653
2654
2655	/**
2656	* Fetches the selector value of a segment register.
2657	*
2658	* @returns The selector value.
2659	* @param pIemCpu The per CPU data.
2660	* @param iSegReg The segment register.
2661	*/
2662	static uint16_t iemSRegFetchU16(PIEMCPU pIemCpu, uint8_t iSegReg)
2663	{
2664	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2665	switch (iSegReg)
2666	{
2667	case X86_SREG_ES: return pCtx->es;
2668	case X86_SREG_CS: return pCtx->cs;
2669	case X86_SREG_SS: return pCtx->ss;
2670	case X86_SREG_DS: return pCtx->ds;
2671	case X86_SREG_FS: return pCtx->fs;
2672	case X86_SREG_GS: return pCtx->gs;
2673	}
2674	AssertFailedReturn(0xffff);
2675	}
2676
2677
2678	/**
2679	* Gets a reference (pointer) to the specified general register.
2680	*
2681	* @returns Register reference.
2682	* @param pIemCpu The per CPU data.
2683	* @param iReg The general register.
2684	*/
2685	static void *iemGRegRef(PIEMCPU pIemCpu, uint8_t iReg)
2686	{
2687	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2688	switch (iReg)
2689	{
2690	case X86_GREG_xAX: return &pCtx->rax;
2691	case X86_GREG_xCX: return &pCtx->rcx;
2692	case X86_GREG_xDX: return &pCtx->rdx;
2693	case X86_GREG_xBX: return &pCtx->rbx;
2694	case X86_GREG_xSP: return &pCtx->rsp;
2695	case X86_GREG_xBP: return &pCtx->rbp;
2696	case X86_GREG_xSI: return &pCtx->rsi;
2697	case X86_GREG_xDI: return &pCtx->rdi;
2698	case X86_GREG_x8: return &pCtx->r8;
2699	case X86_GREG_x9: return &pCtx->r9;
2700	case X86_GREG_x10: return &pCtx->r10;
2701	case X86_GREG_x11: return &pCtx->r11;
2702	case X86_GREG_x12: return &pCtx->r12;
2703	case X86_GREG_x13: return &pCtx->r13;
2704	case X86_GREG_x14: return &pCtx->r14;
2705	case X86_GREG_x15: return &pCtx->r15;
2706	}
2707	AssertFailedReturn(NULL);
2708	}
2709
2710
2711	/**
2712	* Gets a reference (pointer) to the specified 8-bit general register.
2713	*
2714	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
2715	*
2716	* @returns Register reference.
2717	* @param pIemCpu The per CPU data.
2718	* @param iReg The register.
2719	*/
2720	static uint8_t *iemGRegRefU8(PIEMCPU pIemCpu, uint8_t iReg)
2721	{
2722	if (pIemCpu->fPrefixes & IEM_OP_PRF_REX)
2723	return (uint8_t *)iemGRegRef(pIemCpu, iReg);
2724
2725	uint8_t pu8Reg = (uint8_t )iemGRegRef(pIemCpu, iReg & 3);
2726	if (iReg >= 4)
2727	pu8Reg++;
2728	return pu8Reg;
2729	}
2730
2731
2732	/**
2733	* Fetches the value of a 8-bit general register.
2734	*
2735	* @returns The register value.
2736	* @param pIemCpu The per CPU data.
2737	* @param iReg The register.
2738	*/
2739	static uint8_t iemGRegFetchU8(PIEMCPU pIemCpu, uint8_t iReg)
2740	{
2741	uint8_t const *pbSrc = iemGRegRefU8(pIemCpu, iReg);
2742	return *pbSrc;
2743	}
2744
2745
2746	/**
2747	* Fetches the value of a 16-bit general register.
2748	*
2749	* @returns The register value.
2750	* @param pIemCpu The per CPU data.
2751	* @param iReg The register.
2752	*/
2753	static uint16_t iemGRegFetchU16(PIEMCPU pIemCpu, uint8_t iReg)
2754	{
2755	return (uint16_t )iemGRegRef(pIemCpu, iReg);
2756	}
2757
2758
2759	/**
2760	* Fetches the value of a 32-bit general register.
2761	*
2762	* @returns The register value.
2763	* @param pIemCpu The per CPU data.
2764	* @param iReg The register.
2765	*/
2766	static uint32_t iemGRegFetchU32(PIEMCPU pIemCpu, uint8_t iReg)
2767	{
2768	return (uint32_t )iemGRegRef(pIemCpu, iReg);
2769	}
2770
2771
2772	/**
2773	* Fetches the value of a 64-bit general register.
2774	*
2775	* @returns The register value.
2776	* @param pIemCpu The per CPU data.
2777	* @param iReg The register.
2778	*/
2779	static uint64_t iemGRegFetchU64(PIEMCPU pIemCpu, uint8_t iReg)
2780	{
2781	return (uint64_t )iemGRegRef(pIemCpu, iReg);
2782	}
2783
2784
2785	/**
2786	* Is the FPU state in FXSAVE format or not.
2787	*
2788	* @returns true if it is, false if it's in FNSAVE.
2789	* @param pVCpu The virtual CPU handle.
2790	*/
2791	DECLINLINE(bool) iemFRegIsFxSaveFormat(PIEMCPU pIemCpu)
2792	{
2793	#ifdef RT_ARCH_AMD64
2794	NOREF(pIemCpu);
2795	return true;
2796	#else
2797	NOREF(pIemCpu); /// @todo return pVCpu->pVMR3->cpum.s.CPUFeatures.edx.u1FXSR;
2798	return true;
2799	#endif
2800	}
2801
2802
2803	/**
2804	* Adds a 8-bit signed jump offset to RIP/EIP/IP.
2805	*
2806	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2807	* segment limit.
2808	*
2809	* @param pIemCpu The per CPU data.
2810	* @param offNextInstr The offset of the next instruction.
2811	*/
2812	static VBOXSTRICTRC iemRegRipRelativeJumpS8(PIEMCPU pIemCpu, int8_t offNextInstr)
2813	{
2814	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2815	switch (pIemCpu->enmEffOpSize)
2816	{
2817	case IEMMODE_16BIT:
2818	{
2819	uint16_t uNewIp = pCtx->ip + offNextInstr + pIemCpu->offOpcode;
2820	if ( uNewIp > pCtx->csHid.u32Limit
2821	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
2822	return iemRaiseGeneralProtectionFault0(pIemCpu);
2823	pCtx->rip = uNewIp;
2824	break;
2825	}
2826
2827	case IEMMODE_32BIT:
2828	{
2829	Assert(pCtx->rip <= UINT32_MAX);
2830	Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2831
2832	uint32_t uNewEip = pCtx->eip + offNextInstr + pIemCpu->offOpcode;
2833	if (uNewEip > pCtx->csHid.u32Limit)
2834	return iemRaiseGeneralProtectionFault0(pIemCpu);
2835	pCtx->rip = uNewEip;
2836	break;
2837	}
2838
2839	case IEMMODE_64BIT:
2840	{
2841	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2842
2843	uint64_t uNewRip = pCtx->rip + offNextInstr + pIemCpu->offOpcode;
2844	if (!IEM_IS_CANONICAL(uNewRip))
2845	return iemRaiseGeneralProtectionFault0(pIemCpu);
2846	pCtx->rip = uNewRip;
2847	break;
2848	}
2849
2850	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2851	}
2852
2853	return VINF_SUCCESS;
2854	}
2855
2856
2857	/**
2858	* Adds a 16-bit signed jump offset to RIP/EIP/IP.
2859	*
2860	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2861	* segment limit.
2862	*
2863	* @returns Strict VBox status code.
2864	* @param pIemCpu The per CPU data.
2865	* @param offNextInstr The offset of the next instruction.
2866	*/
2867	static VBOXSTRICTRC iemRegRipRelativeJumpS16(PIEMCPU pIemCpu, int16_t offNextInstr)
2868	{
2869	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2870	Assert(pIemCpu->enmEffOpSize == IEMMODE_16BIT);
2871
2872	uint16_t uNewIp = pCtx->ip + offNextInstr + pIemCpu->offOpcode;
2873	if ( uNewIp > pCtx->csHid.u32Limit
2874	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
2875	return iemRaiseGeneralProtectionFault0(pIemCpu);
2876	/** @todo Test 16-bit jump in 64-bit mode. */
2877	pCtx->rip = uNewIp;
2878
2879	return VINF_SUCCESS;
2880	}
2881
2882
2883	/**
2884	* Adds a 32-bit signed jump offset to RIP/EIP/IP.
2885	*
2886	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2887	* segment limit.
2888	*
2889	* @returns Strict VBox status code.
2890	* @param pIemCpu The per CPU data.
2891	* @param offNextInstr The offset of the next instruction.
2892	*/
2893	static VBOXSTRICTRC iemRegRipRelativeJumpS32(PIEMCPU pIemCpu, int32_t offNextInstr)
2894	{
2895	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2896	Assert(pIemCpu->enmEffOpSize != IEMMODE_16BIT);
2897
2898	if (pIemCpu->enmEffOpSize == IEMMODE_32BIT)
2899	{
2900	Assert(pCtx->rip <= UINT32_MAX); Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2901
2902	uint32_t uNewEip = pCtx->eip + offNextInstr + pIemCpu->offOpcode;
2903	if (uNewEip > pCtx->csHid.u32Limit)
2904	return iemRaiseGeneralProtectionFault0(pIemCpu);
2905	pCtx->rip = uNewEip;
2906	}
2907	else
2908	{
2909	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2910
2911	uint64_t uNewRip = pCtx->rip + offNextInstr + pIemCpu->offOpcode;
2912	if (!IEM_IS_CANONICAL(uNewRip))
2913	return iemRaiseGeneralProtectionFault0(pIemCpu);
2914	pCtx->rip = uNewRip;
2915	}
2916	return VINF_SUCCESS;
2917	}
2918
2919
2920	/**
2921	* Performs a near jump to the specified address.
2922	*
2923	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2924	* segment limit.
2925	*
2926	* @param pIemCpu The per CPU data.
2927	* @param uNewRip The new RIP value.
2928	*/
2929	static VBOXSTRICTRC iemRegRipJump(PIEMCPU pIemCpu, uint64_t uNewRip)
2930	{
2931	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2932	switch (pIemCpu->enmEffOpSize)
2933	{
2934	case IEMMODE_16BIT:
2935	{
2936	Assert(uNewRip <= UINT16_MAX);
2937	if ( uNewRip > pCtx->csHid.u32Limit
2938	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
2939	return iemRaiseGeneralProtectionFault0(pIemCpu);
2940	/** @todo Test 16-bit jump in 64-bit mode. */
2941	pCtx->rip = uNewRip;
2942	break;
2943	}
2944
2945	case IEMMODE_32BIT:
2946	{
2947	Assert(uNewRip <= UINT32_MAX);
2948	Assert(pCtx->rip <= UINT32_MAX);
2949	Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2950
2951	if (uNewRip > pCtx->csHid.u32Limit)
2952	return iemRaiseGeneralProtectionFault0(pIemCpu);
2953	pCtx->rip = uNewRip;
2954	break;
2955	}
2956
2957	case IEMMODE_64BIT:
2958	{
2959	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2960
2961	if (!IEM_IS_CANONICAL(uNewRip))
2962	return iemRaiseGeneralProtectionFault0(pIemCpu);
2963	pCtx->rip = uNewRip;
2964	break;
2965	}
2966
2967	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2968	}
2969
2970	return VINF_SUCCESS;
2971	}
2972
2973
2974	/**
2975	* Get the address of the top of the stack.
2976	*
2977	* @param pCtx The CPU context which SP/ESP/RSP should be
2978	* read.
2979	*/
2980	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCCPUMCTX pCtx)
2981	{
2982	if (pCtx->ssHid.Attr.n.u1Long)
2983	return pCtx->rsp;
2984	if (pCtx->ssHid.Attr.n.u1DefBig)
2985	return pCtx->esp;
2986	return pCtx->sp;
2987	}
2988
2989
2990	/**
2991	* Updates the RIP/EIP/IP to point to the next instruction.
2992	*
2993	* @param pIemCpu The per CPU data.
2994	* @param cbInstr The number of bytes to add.
2995	*/
2996	static void iemRegAddToRip(PIEMCPU pIemCpu, uint8_t cbInstr)
2997	{
2998	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2999	switch (pIemCpu->enmCpuMode)
3000	{
3001	case IEMMODE_16BIT:
3002	Assert(pCtx->rip <= UINT16_MAX);
3003	pCtx->eip += cbInstr;
3004	pCtx->eip &= UINT32_C(0xffff);
3005	break;
3006
3007	case IEMMODE_32BIT:
3008	pCtx->eip += cbInstr;
3009	Assert(pCtx->rip <= UINT32_MAX);
3010	break;
3011
3012	case IEMMODE_64BIT:
3013	pCtx->rip += cbInstr;
3014	break;
3015	default: AssertFailed();
3016	}
3017	}
3018
3019
3020	/**
3021	* Updates the RIP/EIP/IP to point to the next instruction.
3022	*
3023	* @param pIemCpu The per CPU data.
3024	*/
3025	static void iemRegUpdateRip(PIEMCPU pIemCpu)
3026	{
3027	return iemRegAddToRip(pIemCpu, pIemCpu->offOpcode);
3028	}
3029
3030
3031	/**
3032	* Adds to the stack pointer.
3033	*
3034	* @param pCtx The CPU context which SP/ESP/RSP should be
3035	* updated.
3036	* @param cbToAdd The number of bytes to add.
3037	*/
3038	DECLINLINE(void) iemRegAddToRsp(PCPUMCTX pCtx, uint8_t cbToAdd)
3039	{
3040	if (pCtx->ssHid.Attr.n.u1Long)
3041	pCtx->rsp += cbToAdd;
3042	else if (pCtx->ssHid.Attr.n.u1DefBig)
3043	pCtx->esp += cbToAdd;
3044	else
3045	pCtx->sp += cbToAdd;
3046	}
3047
3048
3049	/**
3050	* Subtracts from the stack pointer.
3051	*
3052	* @param pCtx The CPU context which SP/ESP/RSP should be
3053	* updated.
3054	* @param cbToSub The number of bytes to subtract.
3055	*/
3056	DECLINLINE(void) iemRegSubFromRsp(PCPUMCTX pCtx, uint8_t cbToSub)
3057	{
3058	if (pCtx->ssHid.Attr.n.u1Long)
3059	pCtx->rsp -= cbToSub;
3060	else if (pCtx->ssHid.Attr.n.u1DefBig)
3061	pCtx->esp -= cbToSub;
3062	else
3063	pCtx->sp -= cbToSub;
3064	}
3065
3066
3067	/**
3068	* Adds to the temporary stack pointer.
3069	*
3070	* @param pTmpRsp The temporary SP/ESP/RSP to update.
3071	* @param cbToAdd The number of bytes to add.
3072	* @param pCtx Where to get the current stack mode.
3073	*/
3074	DECLINLINE(void) iemRegAddToRspEx(PRTUINT64U pTmpRsp, uint8_t cbToAdd, PCCPUMCTX pCtx)
3075	{
3076	if (pCtx->ssHid.Attr.n.u1Long)
3077	pTmpRsp->u += cbToAdd;
3078	else if (pCtx->ssHid.Attr.n.u1DefBig)
3079	pTmpRsp->DWords.dw0 += cbToAdd;
3080	else
3081	pTmpRsp->Words.w0 += cbToAdd;
3082	}
3083
3084
3085	/**
3086	* Subtracts from the temporary stack pointer.
3087	*
3088	* @param pTmpRsp The temporary SP/ESP/RSP to update.
3089	* @param cbToSub The number of bytes to subtract.
3090	* @param pCtx Where to get the current stack mode.
3091	*/
3092	DECLINLINE(void) iemRegSubFromRspEx(PRTUINT64U pTmpRsp, uint8_t cbToSub, PCCPUMCTX pCtx)
3093	{
3094	if (pCtx->ssHid.Attr.n.u1Long)
3095	pTmpRsp->u -= cbToSub;
3096	else if (pCtx->ssHid.Attr.n.u1DefBig)
3097	pTmpRsp->DWords.dw0 -= cbToSub;
3098	else
3099	pTmpRsp->Words.w0 -= cbToSub;
3100	}
3101
3102
3103	/**
3104	* Calculates the effective stack address for a push of the specified size as
3105	* well as the new RSP value (upper bits may be masked).
3106	*
3107	* @returns Effective stack addressf for the push.
3108	* @param pCtx Where to get the current stack mode.
3109	* @param cbItem The size of the stack item to pop.
3110	* @param puNewRsp Where to return the new RSP value.
3111	*/
3112	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCCPUMCTX pCtx, uint8_t cbItem, uint64_t *puNewRsp)
3113	{
3114	RTUINT64U uTmpRsp;
3115	RTGCPTR GCPtrTop;
3116	uTmpRsp.u = pCtx->rsp;
3117
3118	if (pCtx->ssHid.Attr.n.u1Long)
3119	GCPtrTop = uTmpRsp.u -= cbItem;
3120	else if (pCtx->ssHid.Attr.n.u1DefBig)
3121	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
3122	else
3123	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
3124	*puNewRsp = uTmpRsp.u;
3125	return GCPtrTop;
3126	}
3127
3128
3129	/**
3130	* Gets the current stack pointer and calculates the value after a pop of the
3131	* specified size.
3132	*
3133	* @returns Current stack pointer.
3134	* @param pCtx Where to get the current stack mode.
3135	* @param cbItem The size of the stack item to pop.
3136	* @param puNewRsp Where to return the new RSP value.
3137	*/
3138	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCCPUMCTX pCtx, uint8_t cbItem, uint64_t *puNewRsp)
3139	{
3140	RTUINT64U uTmpRsp;
3141	RTGCPTR GCPtrTop;
3142	uTmpRsp.u = pCtx->rsp;
3143
3144	if (pCtx->ssHid.Attr.n.u1Long)
3145	{
3146	GCPtrTop = uTmpRsp.u;
3147	uTmpRsp.u += cbItem;
3148	}
3149	else if (pCtx->ssHid.Attr.n.u1DefBig)
3150	{
3151	GCPtrTop = uTmpRsp.DWords.dw0;
3152	uTmpRsp.DWords.dw0 += cbItem;
3153	}
3154	else
3155	{
3156	GCPtrTop = uTmpRsp.Words.w0;
3157	uTmpRsp.Words.w0 += cbItem;
3158	}
3159	*puNewRsp = uTmpRsp.u;
3160	return GCPtrTop;
3161	}
3162
3163
3164	/**
3165	* Calculates the effective stack address for a push of the specified size as
3166	* well as the new temporary RSP value (upper bits may be masked).
3167	*
3168	* @returns Effective stack addressf for the push.
3169	* @param pTmpRsp The temporary stack pointer. This is updated.
3170	* @param cbItem The size of the stack item to pop.
3171	* @param puNewRsp Where to return the new RSP value.
3172	*/
3173	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PRTUINT64U pTmpRsp, uint8_t cbItem, PCCPUMCTX pCtx)
3174	{
3175	RTGCPTR GCPtrTop;
3176
3177	if (pCtx->ssHid.Attr.n.u1Long)
3178	GCPtrTop = pTmpRsp->u -= cbItem;
3179	else if (pCtx->ssHid.Attr.n.u1DefBig)
3180	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
3181	else
3182	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
3183	return GCPtrTop;
3184	}
3185
3186
3187	/**
3188	* Gets the effective stack address for a pop of the specified size and
3189	* calculates and updates the temporary RSP.
3190	*
3191	* @returns Current stack pointer.
3192	* @param pTmpRsp The temporary stack pointer. This is updated.
3193	* @param pCtx Where to get the current stack mode.
3194	* @param cbItem The size of the stack item to pop.
3195	*/
3196	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PRTUINT64U pTmpRsp, uint8_t cbItem, PCCPUMCTX pCtx)
3197	{
3198	RTGCPTR GCPtrTop;
3199	if (pCtx->ssHid.Attr.n.u1Long)
3200	{
3201	GCPtrTop = pTmpRsp->u;
3202	pTmpRsp->u += cbItem;
3203	}
3204	else if (pCtx->ssHid.Attr.n.u1DefBig)
3205	{
3206	GCPtrTop = pTmpRsp->DWords.dw0;
3207	pTmpRsp->DWords.dw0 += cbItem;
3208	}
3209	else
3210	{
3211	GCPtrTop = pTmpRsp->Words.w0;
3212	pTmpRsp->Words.w0 += cbItem;
3213	}
3214	return GCPtrTop;
3215	}
3216
3217
3218	/**
3219	* Checks if an Intel CPUID feature bit is set.
3220	*
3221	* @returns true / false.
3222	*
3223	* @param pIemCpu The IEM per CPU data.
3224	* @param fEdx The EDX bit to test, or 0 if ECX.
3225	* @param fEcx The ECX bit to test, or 0 if EDX.
3226	* @remarks Used via IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX,
3227	* IEM_IS_INTEL_CPUID_FEATURE_PRESENT_ECX and others.
3228	*/
3229	static bool iemRegIsIntelCpuIdFeaturePresent(PIEMCPU pIemCpu, uint32_t fEdx, uint32_t fEcx)
3230	{
3231	uint32_t uEax, uEbx, uEcx, uEdx;
3232	CPUMGetGuestCpuId(IEMCPU_TO_VMCPU(pIemCpu), 0x00000001, &uEax, &uEbx, &uEcx, &uEdx);
3233	return (fEcx && (uEcx & fEcx))
3234	\|\| (fEdx && (uEdx & fEdx));
3235	}
3236
3237
3238	/**
3239	* Checks if an AMD CPUID feature bit is set.
3240	*
3241	* @returns true / false.
3242	*
3243	* @param pIemCpu The IEM per CPU data.
3244	* @param fEdx The EDX bit to test, or 0 if ECX.
3245	* @param fEcx The ECX bit to test, or 0 if EDX.
3246	* @remarks Used via IEM_IS_AMD_CPUID_FEATURE_PRESENT_EDX,
3247	* IEM_IS_AMD_CPUID_FEATURE_PRESENT_ECX and others.
3248	*/
3249	static bool iemRegIsAmdCpuIdFeaturePresent(PIEMCPU pIemCpu, uint32_t fEdx, uint32_t fEcx)
3250	{
3251	uint32_t uEax, uEbx, uEcx, uEdx;
3252	CPUMGetGuestCpuId(IEMCPU_TO_VMCPU(pIemCpu), 0x80000001, &uEax, &uEbx, &uEcx, &uEdx);
3253	return (fEcx && (uEcx & fEcx))
3254	\|\| (fEdx && (uEdx & fEdx));
3255	}
3256
3257	/** @} */
3258
3259
3260	/** @name FPU access and helpers.
3261	*
3262	* @{
3263	*/
3264
3265
3266	/**
3267	* Hook for preparing to use the host FPU.
3268	*
3269	* This is necessary in ring-0 and raw-mode context.
3270	*
3271	* @param pIemCpu The IEM per CPU data.
3272	*/
3273	DECLINLINE(void) iemFpuPrepareUsage(PIEMCPU pIemCpu)
3274	{
3275	#ifdef IN_RING3
3276	NOREF(pIemCpu);
3277	#else
3278	# error "Implement me"
3279	#endif
3280	}
3281
3282
3283	/**
3284	* Stores a QNaN value into a FPU register.
3285	*
3286	* @param pReg Pointer to the register.
3287	*/
3288	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
3289	{
3290	pReg->au32[0] = UINT32_C(0x00000000);
3291	pReg->au32[1] = UINT32_C(0xc0000000);
3292	pReg->au16[4] = UINT16_C(0xffff);
3293	}
3294
3295
3296	/**
3297	* Updates the FOP, FPU.CS and FPUIP registers.
3298	*
3299	* @param pIemCpu The IEM per CPU data.
3300	* @param pCtx The CPU context.
3301	*/
3302	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PIEMCPU pIemCpu, PCPUMCTX pCtx)
3303	{
3304	pCtx->fpu.FOP = pIemCpu->abOpcode[pIemCpu->offFpuOpcode]
3305	\| ((uint16_t)(pIemCpu->abOpcode[pIemCpu->offFpuOpcode - 1] & 0x7) << 8);
3306	/** @todo FPU.CS and FPUIP needs to be kept seperately. */
3307	if (IEM_IS_REAL_OR_V86_MODE(pIemCpu))
3308	{
3309	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
3310	* happens in real mode here based on the fnsave and fnstenv images. */
3311	pCtx->fpu.CS = 0;
3312	pCtx->fpu.FPUIP = pCtx->eip \| ((uint32_t)pCtx->cs << 4);
3313	}
3314	else
3315	{
3316	pCtx->fpu.CS = pCtx->cs;
3317	pCtx->fpu.FPUIP = pCtx->rip;
3318	}
3319	}
3320
3321
3322	/**
3323	* Updates the FPU.DS and FPUDP registers.
3324	*
3325	* @param pIemCpu The IEM per CPU data.
3326	* @param pCtx The CPU context.
3327	* @param iEffSeg The effective segment register.
3328	* @param GCPtrEff The effective address relative to @a iEffSeg.
3329	*/
3330	DECLINLINE(void) iemFpuUpdateDP(PIEMCPU pIemCpu, PCPUMCTX pCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3331	{
3332	RTSEL sel;
3333	switch (iEffSeg)
3334	{
3335	case X86_SREG_DS: sel = pCtx->ds; break;
3336	case X86_SREG_SS: sel = pCtx->ss; break;
3337	case X86_SREG_CS: sel = pCtx->cs; break;
3338	case X86_SREG_ES: sel = pCtx->es; break;
3339	case X86_SREG_FS: sel = pCtx->fs; break;
3340	case X86_SREG_GS: sel = pCtx->gs; break;
3341	default:
3342	AssertMsgFailed(("%d\n", iEffSeg));
3343	sel = pCtx->ds;
3344	}
3345	/** @todo FPU.DS and FPUDP needs to be kept seperately. */
3346	if (IEM_IS_REAL_OR_V86_MODE(pIemCpu))
3347	{
3348	pCtx->fpu.DS = 0;
3349	pCtx->fpu.FPUDP = (uint32_t)GCPtrEff \| ((uint32_t)sel << 4);
3350	}
3351	else
3352	{
3353	pCtx->fpu.DS = sel;
3354	pCtx->fpu.FPUDP = GCPtrEff;
3355	}
3356	}
3357
3358
3359	/**
3360	* Rotates the stack registers in the push direction.
3361	*
3362	* @param pCtx The CPU context.
3363	* @remarks This is a complete waste of time, but fxsave stores the registers in
3364	* stack order.
3365	*/
3366	DECLINLINE(void) iemFpuRotateStackPush(PCPUMCTX pCtx)
3367	{
3368	RTFLOAT80U r80Tmp = pCtx->fpu.aRegs[7].r80;
3369	pCtx->fpu.aRegs[7].r80 = pCtx->fpu.aRegs[6].r80;
3370	pCtx->fpu.aRegs[6].r80 = pCtx->fpu.aRegs[5].r80;
3371	pCtx->fpu.aRegs[5].r80 = pCtx->fpu.aRegs[4].r80;
3372	pCtx->fpu.aRegs[4].r80 = pCtx->fpu.aRegs[3].r80;
3373	pCtx->fpu.aRegs[3].r80 = pCtx->fpu.aRegs[2].r80;
3374	pCtx->fpu.aRegs[2].r80 = pCtx->fpu.aRegs[1].r80;
3375	pCtx->fpu.aRegs[1].r80 = pCtx->fpu.aRegs[0].r80;
3376	pCtx->fpu.aRegs[0].r80 = r80Tmp;
3377	}
3378
3379
3380	/**
3381	* Rotates the stack registers in the pop direction.
3382	*
3383	* @param pCtx The CPU context.
3384	* @remarks This is a complete waste of time, but fxsave stores the registers in
3385	* stack order.
3386	*/
3387	DECLINLINE(void) iemFpuRotateStackPop(PCPUMCTX pCtx)
3388	{
3389	RTFLOAT80U r80Tmp = pCtx->fpu.aRegs[0].r80;
3390	pCtx->fpu.aRegs[0].r80 = pCtx->fpu.aRegs[1].r80;
3391	pCtx->fpu.aRegs[1].r80 = pCtx->fpu.aRegs[2].r80;
3392	pCtx->fpu.aRegs[2].r80 = pCtx->fpu.aRegs[3].r80;
3393	pCtx->fpu.aRegs[3].r80 = pCtx->fpu.aRegs[4].r80;
3394	pCtx->fpu.aRegs[4].r80 = pCtx->fpu.aRegs[5].r80;
3395	pCtx->fpu.aRegs[5].r80 = pCtx->fpu.aRegs[6].r80;
3396	pCtx->fpu.aRegs[6].r80 = pCtx->fpu.aRegs[7].r80;
3397	pCtx->fpu.aRegs[7].r80 = r80Tmp;
3398	}
3399
3400
3401	/**
3402	* Updates FSW and pushes a FPU result onto the FPU stack if no pending
3403	* exception prevents it.
3404	*
3405	* @param pIemCpu The IEM per CPU data.
3406	* @param pResult The FPU operation result to push.
3407	* @param pCtx The CPU context.
3408	*/
3409	static void iemFpuMaybePushResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult, PCPUMCTX pCtx)
3410	{
3411	/* Update FSW and bail if there are pending exceptions afterwards. */
3412	uint16_t fFsw = pCtx->fpu.FSW & ~X86_FSW_C_MASK;
3413	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
3414	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
3415	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
3416	{
3417	pCtx->fpu.FSW = fFsw;
3418	return;
3419	}
3420
3421	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
3422	if (!(pCtx->fpu.FTW & RT_BIT(iNewTop)))
3423	{
3424	/* All is fine, push the actual value. */
3425	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3426	pCtx->fpu.aRegs[7].r80 = pResult->r80Result;
3427	}
3428	else if (pCtx->fpu.FCW & X86_FCW_IM)
3429	{
3430	/* Masked stack overflow, push QNaN. */
3431	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
3432	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3433	}
3434	else
3435	{
3436	/* Raise stack overflow, don't push anything. */
3437	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
3438	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
3439	return;
3440	}
3441
3442	fFsw &= ~X86_FSW_TOP_MASK;
3443	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
3444	pCtx->fpu.FSW = fFsw;
3445
3446	iemFpuRotateStackPush(pCtx);
3447	}
3448
3449
3450	/**
3451	* Stores a result in a FPU register and updates the FSW and FTW.
3452	*
3453	* @param pIemCpu The IEM per CPU data.
3454	* @param pResult The result to store.
3455	* @param iStReg Which FPU register to store it in.
3456	* @param pCtx The CPU context.
3457	*/
3458	static void iemFpuStoreResultOnly(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg, PCPUMCTX pCtx)
3459	{
3460	Assert(iStReg < 8);
3461	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3462	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3463	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_TOP_MASK;
3464	pCtx->fpu.FTW \|= RT_BIT(iReg);
3465	pCtx->fpu.aRegs[iStReg].r80 = pResult->r80Result;
3466	}
3467
3468
3469	/**
3470	* Only updates the FPU status word (FSW) with the result of the current
3471	* instruction.
3472	*
3473	* @param pCtx The CPU context.
3474	* @param u16FSW The FSW output of the current instruction.
3475	*/
3476	static void iemFpuUpdateFSWOnly(PCPUMCTX pCtx, uint16_t u16FSW)
3477	{
3478	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3479	pCtx->fpu.FSW \|= u16FSW & ~X86_FSW_TOP_MASK;
3480	}
3481
3482
3483	/**
3484	* Pops one item off the FPU stack if no pending exception prevents it.
3485	*
3486	* @param pCtx The CPU context.
3487	*/
3488	static void iemFpuMaybePopOne(PCPUMCTX pCtx)
3489	{
3490	/* Check pending exceptions. */
3491	uint16_t uFSW = pCtx->fpu.FSW;
3492	if ( (pCtx->fpu.FSW & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
3493	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
3494	return;
3495
3496	/* TOP--. */
3497	uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
3498	uFSW &= ~X86_FSW_TOP_MASK;
3499	uFSW \|= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
3500	pCtx->fpu.FSW = uFSW;
3501
3502	/* Mark the previous ST0 as empty. */
3503	iOldTop >>= X86_FSW_TOP_SHIFT;
3504	pCtx->fpu.FTW &= ~RT_BIT(iOldTop);
3505
3506	/* Rotate the registers. */
3507	iemFpuRotateStackPop(pCtx);
3508	}
3509
3510
3511	/**
3512	* Pushes a FPU result onto the FPU stack if no pending exception prevents it.
3513	*
3514	* @param pIemCpu The IEM per CPU data.
3515	* @param pResult The FPU operation result to push.
3516	*/
3517	static void iemFpuPushResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult)
3518	{
3519	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3520	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3521	iemFpuMaybePushResult(pIemCpu, pResult, pCtx);
3522	}
3523
3524
3525	/**
3526	* Pushes a FPU result onto the FPU stack if no pending exception prevents it,
3527	* and sets FPUDP and FPUDS.
3528	*
3529	* @param pIemCpu The IEM per CPU data.
3530	* @param pResult The FPU operation result to push.
3531	* @param iEffSeg The effective segment register.
3532	* @param GCPtrEff The effective address relative to @a iEffSeg.
3533	*/
3534	static void iemFpuPushResultWithMemOp(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3535	{
3536	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3537	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3538	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3539	iemFpuMaybePushResult(pIemCpu, pResult, pCtx);
3540	}
3541
3542
3543	/**
3544	* Replace ST0 with the first value and push the second onto the FPU stack,
3545	* unless a pending exception prevents it.
3546	*
3547	* @param pIemCpu The IEM per CPU data.
3548	* @param pResult The FPU operation result to store and push.
3549	*/
3550	static void iemFpuPushResultTwo(PIEMCPU pIemCpu, PIEMFPURESULTTWO pResult)
3551	{
3552	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3553	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3554
3555	/* Update FSW and bail if there are pending exceptions afterwards. */
3556	uint16_t fFsw = pCtx->fpu.FSW & ~X86_FSW_C_MASK;
3557	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
3558	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
3559	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
3560	{
3561	pCtx->fpu.FSW = fFsw;
3562	return;
3563	}
3564
3565	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
3566	if (!(pCtx->fpu.FTW & RT_BIT(iNewTop)))
3567	{
3568	/* All is fine, push the actual value. */
3569	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3570	pCtx->fpu.aRegs[0].r80 = pResult->r80Result1;
3571	pCtx->fpu.aRegs[7].r80 = pResult->r80Result2;
3572	}
3573	else if (pCtx->fpu.FCW & X86_FCW_IM)
3574	{
3575	/* Masked stack overflow, push QNaN. */
3576	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
3577	iemFpuStoreQNan(&pCtx->fpu.aRegs[0].r80);
3578	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3579	}
3580	else
3581	{
3582	/* Raise stack overflow, don't push anything. */
3583	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
3584	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
3585	return;
3586	}
3587
3588	fFsw &= ~X86_FSW_TOP_MASK;
3589	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
3590	pCtx->fpu.FSW = fFsw;
3591
3592	iemFpuRotateStackPush(pCtx);
3593	}
3594
3595
3596	/**
3597	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
3598	* FOP.
3599	*
3600	* @param pIemCpu The IEM per CPU data.
3601	* @param pResult The result to store.
3602	* @param iStReg Which FPU register to store it in.
3603	* @param pCtx The CPU context.
3604	*/
3605	static void iemFpuStoreResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg)
3606	{
3607	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3608	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3609	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3610	}
3611
3612
3613	/**
3614	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
3615	* FOP, and then pops the stack.
3616	*
3617	* @param pIemCpu The IEM per CPU data.
3618	* @param pResult The result to store.
3619	* @param iStReg Which FPU register to store it in.
3620	* @param pCtx The CPU context.
3621	*/
3622	static void iemFpuStoreResultThenPop(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg)
3623	{
3624	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3625	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3626	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3627	iemFpuMaybePopOne(pCtx);
3628	}
3629
3630
3631	/**
3632	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
3633	* FPUDP, and FPUDS.
3634	*
3635	* @param pIemCpu The IEM per CPU data.
3636	* @param pResult The result to store.
3637	* @param iStReg Which FPU register to store it in.
3638	* @param pCtx The CPU context.
3639	* @param iEffSeg The effective memory operand selector register.
3640	* @param GCPtrEff The effective memory operand offset.
3641	*/
3642	static void iemFpuStoreResultWithMemOp(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3643	{
3644	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3645	iemFpuUpdateDP(pIemCpu, pIemCpu->CTX_SUFF(pCtx), iEffSeg, GCPtrEff);
3646	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3647	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3648	}
3649
3650
3651	/**
3652	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
3653	* FPUDP, and FPUDS, and then pops the stack.
3654	*
3655	* @param pIemCpu The IEM per CPU data.
3656	* @param pResult The result to store.
3657	* @param iStReg Which FPU register to store it in.
3658	* @param pCtx The CPU context.
3659	* @param iEffSeg The effective memory operand selector register.
3660	* @param GCPtrEff The effective memory operand offset.
3661	*/
3662	static void iemFpuStoreResultWithMemOpThenPop(PIEMCPU pIemCpu, PIEMFPURESULT pResult,
3663	uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3664	{
3665	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3666	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3667	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3668	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3669	iemFpuMaybePopOne(pCtx);
3670	}
3671
3672
3673	/**
3674	* Updates the FOP, FPUIP, and FPUCS. For FNOP.
3675	*
3676	* @param pIemCpu The IEM per CPU data.
3677	*/
3678	static void iemFpuUpdateOpcodeAndIp(PIEMCPU pIemCpu)
3679	{
3680	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pIemCpu->CTX_SUFF(pCtx));
3681	}
3682
3683
3684	/**
3685	* Updates the FSW, FOP, FPUIP, and FPUCS.
3686	*
3687	* @param pIemCpu The IEM per CPU data.
3688	* @param u16FSW The FSW from the current instruction.
3689	*/
3690	static void iemFpuUpdateFSW(PIEMCPU pIemCpu, uint16_t u16FSW)
3691	{
3692	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3693	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3694	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3695	}
3696
3697
3698	/**
3699	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
3700	*
3701	* @param pIemCpu The IEM per CPU data.
3702	* @param u16FSW The FSW from the current instruction.
3703	*/
3704	static void iemFpuUpdateFSWThenPop(PIEMCPU pIemCpu, uint16_t u16FSW)
3705	{
3706	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3707	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3708	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3709	iemFpuMaybePopOne(pCtx);
3710	}
3711
3712
3713	/**
3714	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
3715	*
3716	* @param pIemCpu The IEM per CPU data.
3717	* @param u16FSW The FSW from the current instruction.
3718	* @param iEffSeg The effective memory operand selector register.
3719	* @param GCPtrEff The effective memory operand offset.
3720	*/
3721	static void iemFpuUpdateFSWWithMemOp(PIEMCPU pIemCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3722	{
3723	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3724	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3725	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3726	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3727	}
3728
3729
3730	/**
3731	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
3732	*
3733	* @param pIemCpu The IEM per CPU data.
3734	* @param u16FSW The FSW from the current instruction.
3735	*/
3736	static void iemFpuUpdateFSWThenPopPop(PIEMCPU pIemCpu, uint16_t u16FSW)
3737	{
3738	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3739	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3740	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3741	iemFpuMaybePopOne(pCtx);
3742	iemFpuMaybePopOne(pCtx);
3743	}
3744
3745
3746	/**
3747	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
3748	*
3749	* @param pIemCpu The IEM per CPU data.
3750	* @param u16FSW The FSW from the current instruction.
3751	* @param iEffSeg The effective memory operand selector register.
3752	* @param GCPtrEff The effective memory operand offset.
3753	*/
3754	static void iemFpuUpdateFSWWithMemOpThenPop(PIEMCPU pIemCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3755	{
3756	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3757	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3758	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3759	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3760	iemFpuMaybePopOne(pCtx);
3761	}
3762
3763
3764	/**
3765	* Worker routine for raising an FPU stack underflow exception.
3766	*
3767	* @param pIemCpu The IEM per CPU data.
3768	* @param iStReg The stack register being accessed.
3769	* @param pCtx The CPU context.
3770	*/
3771	static void iemFpuStackUnderflowOnly(PIEMCPU pIemCpu, uint8_t iStReg, PCPUMCTX pCtx)
3772	{
3773	Assert(iStReg < 8 \|\| iStReg == UINT8_MAX);
3774	if (pCtx->fpu.FCW & X86_FCW_IM)
3775	{
3776	/* Masked underflow. */
3777	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3778	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF;
3779	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3780	if (iStReg != UINT8_MAX)
3781	{
3782	pCtx->fpu.FTW \|= RT_BIT(iReg);
3783	iemFpuStoreQNan(&pCtx->fpu.aRegs[iStReg].r80);
3784	}
3785	}
3786	else
3787	{
3788	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3789	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
3790	}
3791	}
3792
3793
3794	/**
3795	* Raises a FPU stack underflow exception.
3796	*
3797	* @param pIemCpu The IEM per CPU data.
3798	* @param iStReg The destination register that should be loaded
3799	* with QNaN if \#IS is not masked. Specify
3800	* UINT8_MAX if none (like for fcom).
3801	*/
3802	DECL_NO_INLINE(static, void) iemFpuStackUnderflow(PIEMCPU pIemCpu, uint8_t iStReg)
3803	{
3804	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3805	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3806	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3807	}
3808
3809
3810	DECL_NO_INLINE(static, void)
3811	iemFpuStackUnderflowWithMemOp(PIEMCPU pIemCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3812	{
3813	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3814	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3815	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3816	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3817	}
3818
3819
3820	DECL_NO_INLINE(static, void) iemFpuStackUnderflowThenPop(PIEMCPU pIemCpu, uint8_t iStReg)
3821	{
3822	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3823	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3824	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3825	iemFpuMaybePopOne(pCtx);
3826	}
3827
3828
3829	DECL_NO_INLINE(static, void)
3830	iemFpuStackUnderflowWithMemOpThenPop(PIEMCPU pIemCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3831	{
3832	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3833	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3834	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3835	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3836	iemFpuMaybePopOne(pCtx);
3837	}
3838
3839
3840	DECL_NO_INLINE(static, void) iemFpuStackUnderflowThenPopPop(PIEMCPU pIemCpu)
3841	{
3842	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3843	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3844	iemFpuStackUnderflowOnly(pIemCpu, UINT8_MAX, pCtx);
3845	iemFpuMaybePopOne(pCtx);
3846	iemFpuMaybePopOne(pCtx);
3847	}
3848
3849
3850	DECL_NO_INLINE(static, void)
3851	iemFpuStackPushUnderflow(PIEMCPU pIemCpu)
3852	{
3853	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3854	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3855
3856	if (pCtx->fpu.FCW & X86_FCW_IM)
3857	{
3858	/* Masked overflow - Push QNaN. */
3859	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
3860	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
3861	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF;
3862	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
3863	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3864	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3865	iemFpuRotateStackPush(pCtx);
3866	}
3867	else
3868	{
3869	/* Exception pending - don't change TOP or the register stack. */
3870	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3871	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
3872	}
3873	}
3874
3875
3876	DECL_NO_INLINE(static, void)
3877	iemFpuStackPushUnderflowTwo(PIEMCPU pIemCpu)
3878	{
3879	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3880	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3881
3882	if (pCtx->fpu.FCW & X86_FCW_IM)
3883	{
3884	/* Masked overflow - Push QNaN. */
3885	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
3886	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
3887	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF;
3888	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
3889	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3890	iemFpuStoreQNan(&pCtx->fpu.aRegs[0].r80);
3891	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3892	iemFpuRotateStackPush(pCtx);
3893	}
3894	else
3895	{
3896	/* Exception pending - don't change TOP or the register stack. */
3897	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3898	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
3899	}
3900	}
3901
3902
3903	/**
3904	* Worker routine for raising an FPU stack overflow exception on a push.
3905	*
3906	* @param pIemCpu The IEM per CPU data.
3907	* @param pCtx The CPU context.
3908	*/
3909	static void iemFpuStackPushOverflowOnly(PIEMCPU pIemCpu, PCPUMCTX pCtx)
3910	{
3911	if (pCtx->fpu.FCW & X86_FCW_IM)
3912	{
3913	/* Masked overflow. */
3914	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
3915	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
3916	pCtx->fpu.FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
3917	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
3918	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3919	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3920	iemFpuRotateStackPush(pCtx);
3921	}
3922	else
3923	{
3924	/* Exception pending - don't change TOP or the register stack. */
3925	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3926	pCtx->fpu.FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
3927	}
3928	}
3929
3930
3931	/**
3932	* Raises a FPU stack overflow exception on a push.
3933	*
3934	* @param pIemCpu The IEM per CPU data.
3935	*/
3936	DECL_NO_INLINE(static, void) iemFpuStackPushOverflow(PIEMCPU pIemCpu)
3937	{
3938	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3939	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3940	iemFpuStackPushOverflowOnly(pIemCpu, pCtx);
3941	}
3942
3943
3944	/**
3945	* Raises a FPU stack overflow exception on a push with a memory operand.
3946	*
3947	* @param pIemCpu The IEM per CPU data.
3948	* @param iEffSeg The effective memory operand selector register.
3949	* @param GCPtrEff The effective memory operand offset.
3950	*/
3951	DECL_NO_INLINE(static, void)
3952	iemFpuStackPushOverflowWithMemOp(PIEMCPU pIemCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3953	{
3954	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3955	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3956	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3957	iemFpuStackPushOverflowOnly(pIemCpu, pCtx);
3958	}
3959
3960
3961	static int iemFpuStRegNotEmpty(PIEMCPU pIemCpu, uint8_t iStReg)
3962	{
3963	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3964	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3965	if (pCtx->fpu.FTW & RT_BIT(iReg))
3966	return VINF_SUCCESS;
3967	return VERR_NOT_FOUND;
3968	}
3969
3970
3971	static int iemFpuStRegNotEmptyRef(PIEMCPU pIemCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
3972	{
3973	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3974	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3975	if (pCtx->fpu.FTW & RT_BIT(iReg))
3976	{
3977	*ppRef = &pCtx->fpu.aRegs[iStReg].r80;
3978	return VINF_SUCCESS;
3979	}
3980	return VERR_NOT_FOUND;
3981	}
3982
3983
3984	static int iemFpu2StRegsNotEmptyRef(PIEMCPU pIemCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
3985	uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
3986	{
3987	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3988	uint16_t iTop = X86_FSW_TOP_GET(pCtx->fpu.FSW);
3989	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
3990	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
3991	if ((pCtx->fpu.FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
3992	{
3993	*ppRef0 = &pCtx->fpu.aRegs[iStReg0].r80;
3994	*ppRef1 = &pCtx->fpu.aRegs[iStReg1].r80;
3995	return VINF_SUCCESS;
3996	}
3997	return VERR_NOT_FOUND;
3998	}
3999
4000
4001	static int iemFpu2StRegsNotEmptyRefFirst(PIEMCPU pIemCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
4002	{
4003	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4004	uint16_t iTop = X86_FSW_TOP_GET(pCtx->fpu.FSW);
4005	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
4006	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
4007	if ((pCtx->fpu.FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
4008	{
4009	*ppRef0 = &pCtx->fpu.aRegs[iStReg0].r80;
4010	return VINF_SUCCESS;
4011	}
4012	return VERR_NOT_FOUND;
4013	}
4014
4015
4016	/**
4017	* Updates the FPU exception status after FCW is changed.
4018	*
4019	* @param pCtx The CPU context.
4020	*/
4021	static void iemFpuRecalcExceptionStatus(PCPUMCTX pCtx)
4022	{
4023	uint16_t u16Fsw = pCtx->fpu.FSW;
4024	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pCtx->fpu.FCW & X86_FCW_XCPT_MASK))
4025	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
4026	else
4027	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
4028	pCtx->fpu.FSW = u16Fsw;
4029	}
4030
4031
4032	/**
4033	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
4034	*
4035	* @returns The full FTW.
4036	* @param pCtx The CPU state.
4037	*/
4038	static uint16_t iemFpuCalcFullFtw(PCCPUMCTX pCtx)
4039	{
4040	uint8_t const u8Ftw = (uint8_t)pCtx->fpu.FTW;
4041	uint16_t u16Ftw = 0;
4042	unsigned const iTop = X86_FSW_TOP_GET(pCtx->fpu.FSW);
4043	for (unsigned iSt = 0; iSt < 8; iSt++)
4044	{
4045	unsigned const iReg = (iSt + iTop) & 7;
4046	if (!(u8Ftw & RT_BIT(iReg)))
4047	u16Ftw \|= 3 << (iReg * 2); /* empty */
4048	else
4049	{
4050	uint16_t uTag;
4051	PCRTFLOAT80U const pr80Reg = &pCtx->fpu.aRegs[iSt].r80;
4052	if (pr80Reg->s.uExponent == 0x7fff)
4053	uTag = 2; /* Exponent is all 1's => Special. */
4054	else if (pr80Reg->s.uExponent == 0x0000)
4055	{
4056	if (pr80Reg->s.u64Mantissa == 0x0000)
4057	uTag = 1; /* All bits are zero => Zero. */
4058	else
4059	uTag = 2; /* Must be special. */
4060	}
4061	else if (pr80Reg->s.u64Mantissa & RT_BIT_64(63)) /* The J bit. */
4062	uTag = 0; /* Valid. */
4063	else
4064	uTag = 2; /* Must be special. */
4065
4066	u16Ftw \|= uTag << (iReg * 2); /* empty */
4067	}
4068	}
4069
4070	return u16Ftw;
4071	}
4072
4073
4074	/**
4075	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
4076	*
4077	* @returns The compressed FTW.
4078	* @param u16FullFtw The full FTW to convert.
4079	*/
4080	static uint16_t iemFpuCompressFtw(uint16_t u16FullFtw)
4081	{
4082	uint8_t u8Ftw = 0;
4083	for (unsigned i = 0; i < 8; i++)
4084	{
4085	if ((u16FullFtw & 3) != 3 /empty/)
4086	u8Ftw \|= RT_BIT(i);
4087	u16FullFtw >>= 2;
4088	}
4089
4090	return u8Ftw;
4091	}
4092
4093	/** @} */
4094
4095
4096	/** @name Memory access.
4097	*
4098	* @{
4099	*/
4100
4101
4102	/**
4103	* Checks if the given segment can be written to, raise the appropriate
4104	* exception if not.
4105	*
4106	* @returns VBox strict status code.
4107	*
4108	* @param pIemCpu The IEM per CPU data.
4109	* @param pHid Pointer to the hidden register.
4110	* @param iSegReg The register number.
4111	*/
4112	static VBOXSTRICTRC iemMemSegCheckWriteAccessEx(PIEMCPU pIemCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg)
4113	{
4114	if (!pHid->Attr.n.u1Present)
4115	return iemRaiseSelectorNotPresentBySegReg(pIemCpu, iSegReg);
4116
4117	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
4118	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
4119	&& pIemCpu->enmCpuMode != IEMMODE_64BIT )
4120	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, IEM_ACCESS_DATA_W);
4121
4122	/** @todo DPL/RPL/CPL? */
4123
4124	return VINF_SUCCESS;
4125	}
4126
4127
4128	/**
4129	* Checks if the given segment can be read from, raise the appropriate
4130	* exception if not.
4131	*
4132	* @returns VBox strict status code.
4133	*
4134	* @param pIemCpu The IEM per CPU data.
4135	* @param pHid Pointer to the hidden register.
4136	* @param iSegReg The register number.
4137	*/
4138	static VBOXSTRICTRC iemMemSegCheckReadAccessEx(PIEMCPU pIemCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg)
4139	{
4140	if (!pHid->Attr.n.u1Present)
4141	return iemRaiseSelectorNotPresentBySegReg(pIemCpu, iSegReg);
4142
4143	if ( (pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE
4144	&& pIemCpu->enmCpuMode != IEMMODE_64BIT )
4145	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, IEM_ACCESS_DATA_R);
4146
4147	/** @todo DPL/RPL/CPL? */
4148
4149	return VINF_SUCCESS;
4150	}
4151
4152
4153	/**
4154	* Applies the segment limit, base and attributes.
4155	*
4156	* This may raise a \#GP or \#SS.
4157	*
4158	* @returns VBox strict status code.
4159	*
4160	* @param pIemCpu The IEM per CPU data.
4161	* @param fAccess The kind of access which is being performed.
4162	* @param iSegReg The index of the segment register to apply.
4163	* This is UINT8_MAX if none (for IDT, GDT, LDT,
4164	* TSS, ++).
4165	* @param pGCPtrMem Pointer to the guest memory address to apply
4166	* segmentation to. Input and output parameter.
4167	*/
4168	static VBOXSTRICTRC iemMemApplySegment(PIEMCPU pIemCpu, uint32_t fAccess, uint8_t iSegReg,
4169	size_t cbMem, PRTGCPTR pGCPtrMem)
4170	{
4171	if (iSegReg == UINT8_MAX)
4172	return VINF_SUCCESS;
4173
4174	PCPUMSELREGHID pSel = iemSRegGetHid(pIemCpu, iSegReg);
4175	switch (pIemCpu->enmCpuMode)
4176	{
4177	case IEMMODE_16BIT:
4178	case IEMMODE_32BIT:
4179	{
4180	RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
4181	RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
4182
4183	Assert(pSel->Attr.n.u1Present);
4184	Assert(pSel->Attr.n.u1DescType);
4185	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
4186	{
4187	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
4188	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
4189	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, fAccess);
4190
4191	if (!IEM_IS_REAL_OR_V86_MODE(pIemCpu))
4192	{
4193	/** @todo CPL check. */
4194	}
4195
4196	/*
4197	* There are two kinds of data selectors, normal and expand down.
4198	*/
4199	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
4200	{
4201	if ( GCPtrFirst32 > pSel->u32Limit
4202	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
4203	return iemRaiseSelectorBounds(pIemCpu, iSegReg, fAccess);
4204
4205	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
4206	}
4207	else
4208	{
4209	/** @todo implement expand down segments. */
4210	AssertFailed(/** @todo implement this */);
4211	return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
4212	}
4213	}
4214	else
4215	{
4216
4217	/*
4218	* Code selector and usually be used to read thru, writing is
4219	* only permitted in real and V8086 mode.
4220	*/
4221	if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
4222	\|\| ( (fAccess & IEM_ACCESS_TYPE_READ)
4223	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
4224	&& !IEM_IS_REAL_OR_V86_MODE(pIemCpu) )
4225	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, fAccess);
4226
4227	if ( GCPtrFirst32 > pSel->u32Limit
4228	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
4229	return iemRaiseSelectorBounds(pIemCpu, iSegReg, fAccess);
4230
4231	if (!IEM_IS_REAL_OR_V86_MODE(pIemCpu))
4232	{
4233	/** @todo CPL check. */
4234	}
4235
4236	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
4237	}
4238	return VINF_SUCCESS;
4239	}
4240
4241	case IEMMODE_64BIT:
4242	if (iSegReg == X86_SREG_GS \|\| iSegReg == X86_SREG_FS)
4243	*pGCPtrMem += pSel->u64Base;
4244	return VINF_SUCCESS;
4245
4246	default:
4247	AssertFailedReturn(VERR_INTERNAL_ERROR_5);
4248	}
4249	}
4250
4251
4252	/**
4253	* Translates a virtual address to a physical physical address and checks if we
4254	* can access the page as specified.
4255	*
4256	* @param pIemCpu The IEM per CPU data.
4257	* @param GCPtrMem The virtual address.
4258	* @param fAccess The intended access.
4259	* @param pGCPhysMem Where to return the physical address.
4260	*/
4261	static VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PIEMCPU pIemCpu, RTGCPTR GCPtrMem, uint32_t fAccess,
4262	PRTGCPHYS pGCPhysMem)
4263	{
4264	/** @todo Need a different PGM interface here. We're currently using
4265	* generic / REM interfaces. this won't cut it for R0 & RC. */
4266	RTGCPHYS GCPhys;
4267	uint64_t fFlags;
4268	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrMem, &fFlags, &GCPhys);
4269	if (RT_FAILURE(rc))
4270	{
4271	/** @todo Check unassigned memory in unpaged mode. */
4272	/** @todo Reserved bits in page tables. Requires new PGM interface. */
4273	*pGCPhysMem = NIL_RTGCPHYS;
4274	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess, rc);
4275	}
4276
4277	/* If the page is writable and does not have the no-exec bit set, all
4278	access is allowed. Otherwise we'll have to check more carefully... */
4279	if ((fFlags & (X86_PTE_RW \| X86_PTE_US \| X86_PTE_PAE_NX)) != (X86_PTE_RW \| X86_PTE_US))
4280	{
4281	/* Write to read only memory? */
4282	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
4283	&& !(fFlags & X86_PTE_RW)
4284	&& ( pIemCpu->uCpl != 0
4285	\|\| (pIemCpu->CTX_SUFF(pCtx)->cr0 & X86_CR0_WP)))
4286	{
4287	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
4288	*pGCPhysMem = NIL_RTGCPHYS;
4289	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
4290	}
4291
4292	/* Kernel memory accessed by userland? */
4293	if ( !(fFlags & X86_PTE_US)
4294	&& pIemCpu->uCpl == 3
4295	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
4296	{
4297	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
4298	*pGCPhysMem = NIL_RTGCPHYS;
4299	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
4300	}
4301
4302	/* Executing non-executable memory? */
4303	if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
4304	&& (fFlags & X86_PTE_PAE_NX)
4305	&& (pIemCpu->CTX_SUFF(pCtx)->msrEFER & MSR_K6_EFER_NXE) )
4306	{
4307	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
4308	*pGCPhysMem = NIL_RTGCPHYS;
4309	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE),
4310	VERR_ACCESS_DENIED);
4311	}
4312	}
4313
4314	GCPhys \|= GCPtrMem & PAGE_OFFSET_MASK;
4315	*pGCPhysMem = GCPhys;
4316	return VINF_SUCCESS;
4317	}
4318
4319
4320
4321	/**
4322	* Maps a physical page.
4323	*
4324	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
4325	* @param pIemCpu The IEM per CPU data.
4326	* @param GCPhysMem The physical address.
4327	* @param fAccess The intended access.
4328	* @param ppvMem Where to return the mapping address.
4329	*/
4330	static int iemMemPageMap(PIEMCPU pIemCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem)
4331	{
4332	#ifdef IEM_VERIFICATION_MODE
4333	/* Force the alternative path so we can ignore writes. */
4334	if ((fAccess & IEM_ACCESS_TYPE_WRITE) && !pIemCpu->fNoRem)
4335	return VERR_PGM_PHYS_TLB_CATCH_ALL;
4336	#endif
4337
4338	/*
4339	* If we can map the page without trouble, do a block processing
4340	* until the end of the current page.
4341	*/
4342	/** @todo need some better API. */
4343	return PGMR3PhysTlbGCPhys2Ptr(IEMCPU_TO_VM(pIemCpu),
4344	GCPhysMem,
4345	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
4346	ppvMem);
4347	}
4348
4349
4350	/**
4351	* Unmap a page previously mapped by iemMemPageMap.
4352	*
4353	* This is currently a dummy function.
4354	*
4355	* @param pIemCpu The IEM per CPU data.
4356	* @param GCPhysMem The physical address.
4357	* @param fAccess The intended access.
4358	* @param pvMem What iemMemPageMap returned.
4359	*/
4360	DECLINLINE(void) iemMemPageUnmap(PIEMCPU pIemCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem)
4361	{
4362	NOREF(pIemCpu);
4363	NOREF(GCPhysMem);
4364	NOREF(fAccess);
4365	NOREF(pvMem);
4366	}
4367
4368
4369	/**
4370	* Looks up a memory mapping entry.
4371	*
4372	* @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
4373	* @param pIemCpu The IEM per CPU data.
4374	* @param pvMem The memory address.
4375	* @param fAccess The access to.
4376	*/
4377	DECLINLINE(int) iemMapLookup(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess)
4378	{
4379	fAccess &= IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK;
4380	if ( pIemCpu->aMemMappings[0].pv == pvMem
4381	&& (pIemCpu->aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
4382	return 0;
4383	if ( pIemCpu->aMemMappings[1].pv == pvMem
4384	&& (pIemCpu->aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
4385	return 1;
4386	if ( pIemCpu->aMemMappings[2].pv == pvMem
4387	&& (pIemCpu->aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
4388	return 2;
4389	return VERR_NOT_FOUND;
4390	}
4391
4392
4393	/**
4394	* Finds a free memmap entry when using iNextMapping doesn't work.
4395	*
4396	* @returns Memory mapping index, 1024 on failure.
4397	* @param pIemCpu The IEM per CPU data.
4398	*/
4399	static unsigned iemMemMapFindFree(PIEMCPU pIemCpu)
4400	{
4401	/*
4402	* The easy case.
4403	*/
4404	if (pIemCpu->cActiveMappings == 0)
4405	{
4406	pIemCpu->iNextMapping = 1;
4407	return 0;
4408	}
4409
4410	/* There should be enough mappings for all instructions. */
4411	AssertReturn(pIemCpu->cActiveMappings < RT_ELEMENTS(pIemCpu->aMemMappings), 1024);
4412
4413	for (unsigned i = 0; i < RT_ELEMENTS(pIemCpu->aMemMappings); i++)
4414	if (pIemCpu->aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
4415	return i;
4416
4417	AssertFailedReturn(1024);
4418	}
4419
4420
4421	/**
4422	* Commits a bounce buffer that needs writing back and unmaps it.
4423	*
4424	* @returns Strict VBox status code.
4425	* @param pIemCpu The IEM per CPU data.
4426	* @param iMemMap The index of the buffer to commit.
4427	*/
4428	static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PIEMCPU pIemCpu, unsigned iMemMap)
4429	{
4430	Assert(pIemCpu->aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
4431	Assert(pIemCpu->aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
4432
4433	/*
4434	* Do the writing.
4435	*/
4436	int rc;
4437	if ( !pIemCpu->aMemBbMappings[iMemMap].fUnassigned
4438	&& !IEM_VERIFICATION_ENABLED(pIemCpu))
4439	{
4440	uint16_t const cbFirst = pIemCpu->aMemBbMappings[iMemMap].cbFirst;
4441	uint16_t const cbSecond = pIemCpu->aMemBbMappings[iMemMap].cbSecond;
4442	uint8_t const *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
4443	if (!pIemCpu->fByPassHandlers)
4444	{
4445	rc = PGMPhysWrite(IEMCPU_TO_VM(pIemCpu),
4446	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst,
4447	pbBuf,
4448	cbFirst);
4449	if (cbSecond && rc == VINF_SUCCESS)
4450	rc = PGMPhysWrite(IEMCPU_TO_VM(pIemCpu),
4451	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond,
4452	pbBuf + cbFirst,
4453	cbSecond);
4454	}
4455	else
4456	{
4457	rc = PGMPhysSimpleWriteGCPhys(IEMCPU_TO_VM(pIemCpu),
4458	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst,
4459	pbBuf,
4460	cbFirst);
4461	if (cbSecond && rc == VINF_SUCCESS)
4462	rc = PGMPhysSimpleWriteGCPhys(IEMCPU_TO_VM(pIemCpu),
4463	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond,
4464	pbBuf + cbFirst,
4465	cbSecond);
4466	}
4467	}
4468	else
4469	rc = VINF_SUCCESS;
4470
4471	#ifdef IEM_VERIFICATION_MODE
4472	/*
4473	* Record the write(s).
4474	*/
4475	if (!pIemCpu->fNoRem)
4476	{
4477	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
4478	if (pEvtRec)
4479	{
4480	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
4481	pEvtRec->u.RamWrite.GCPhys = pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst;
4482	pEvtRec->u.RamWrite.cb = pIemCpu->aMemBbMappings[iMemMap].cbFirst;
4483	memcpy(pEvtRec->u.RamWrite.ab, &pIemCpu->aBounceBuffers[iMemMap].ab[0], pIemCpu->aMemBbMappings[iMemMap].cbFirst);
4484	AssertCompile(sizeof(pEvtRec->u.RamWrite.ab) == sizeof(pIemCpu->aBounceBuffers[0].ab));
4485	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4486	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4487	}
4488	if (pIemCpu->aMemBbMappings[iMemMap].cbSecond)
4489	{
4490	pEvtRec = iemVerifyAllocRecord(pIemCpu);
4491	if (pEvtRec)
4492	{
4493	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
4494	pEvtRec->u.RamWrite.GCPhys = pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond;
4495	pEvtRec->u.RamWrite.cb = pIemCpu->aMemBbMappings[iMemMap].cbSecond;
4496	memcpy(pEvtRec->u.RamWrite.ab,
4497	&pIemCpu->aBounceBuffers[iMemMap].ab[pIemCpu->aMemBbMappings[iMemMap].cbFirst],
4498	pIemCpu->aMemBbMappings[iMemMap].cbSecond);
4499	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4500	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4501	}
4502	}
4503	}
4504	#endif
4505
4506	/*
4507	* Free the mapping entry.
4508	*/
4509	pIemCpu->aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
4510	Assert(pIemCpu->cActiveMappings != 0);
4511	pIemCpu->cActiveMappings--;
4512	return rc;
4513	}
4514
4515
4516	/**
4517	* iemMemMap worker that deals with a request crossing pages.
4518	*/
4519	static VBOXSTRICTRC iemMemBounceBufferMapCrossPage(PIEMCPU pIemCpu, int iMemMap, void **ppvMem,
4520	size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
4521	{
4522	/*
4523	* Do the address translations.
4524	*/
4525	RTGCPHYS GCPhysFirst;
4526	VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrFirst, fAccess, &GCPhysFirst);
4527	if (rcStrict != VINF_SUCCESS)
4528	return rcStrict;
4529
4530	RTGCPHYS GCPhysSecond;
4531	rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrFirst + (cbMem - 1), fAccess, &GCPhysSecond);
4532	if (rcStrict != VINF_SUCCESS)
4533	return rcStrict;
4534	GCPhysSecond &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
4535
4536	/*
4537	* Read in the current memory content if it's a read, execute or partial
4538	* write access.
4539	*/
4540	uint8_t *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
4541	uint32_t const cbFirstPage = PAGE_SIZE - (GCPhysFirst & PAGE_OFFSET_MASK);
4542	uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
4543
4544	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
4545	{
4546	int rc;
4547	if (!pIemCpu->fByPassHandlers)
4548	{
4549	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysFirst, pbBuf, cbFirstPage);
4550	if (rc != VINF_SUCCESS)
4551	return rc;
4552	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage);
4553	if (rc != VINF_SUCCESS)
4554	return rc;
4555	}
4556	else
4557	{
4558	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf, GCPhysFirst, cbFirstPage);
4559	if (rc != VINF_SUCCESS)
4560	return rc;
4561	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
4562	if (rc != VINF_SUCCESS)
4563	return rc;
4564	}
4565
4566	#ifdef IEM_VERIFICATION_MODE
4567	if ( !pIemCpu->fNoRem
4568	&& (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC)) )
4569	{
4570	/*
4571	* Record the reads.
4572	*/
4573	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
4574	if (pEvtRec)
4575	{
4576	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
4577	pEvtRec->u.RamRead.GCPhys = GCPhysFirst;
4578	pEvtRec->u.RamRead.cb = cbFirstPage;
4579	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4580	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4581	}
4582	pEvtRec = iemVerifyAllocRecord(pIemCpu);
4583	if (pEvtRec)
4584	{
4585	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
4586	pEvtRec->u.RamRead.GCPhys = GCPhysSecond;
4587	pEvtRec->u.RamRead.cb = cbSecondPage;
4588	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4589	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4590	}
4591	}
4592	#endif
4593	}
4594	#ifdef VBOX_STRICT
4595	else
4596	memset(pbBuf, 0xcc, cbMem);
4597	#endif
4598	#ifdef VBOX_STRICT
4599	if (cbMem < sizeof(pIemCpu->aBounceBuffers[iMemMap].ab))
4600	memset(pbBuf + cbMem, 0xaa, sizeof(pIemCpu->aBounceBuffers[iMemMap].ab) - cbMem);
4601	#endif
4602
4603	/*
4604	* Commit the bounce buffer entry.
4605	*/
4606	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
4607	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
4608	pIemCpu->aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
4609	pIemCpu->aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
4610	pIemCpu->aMemBbMappings[iMemMap].fUnassigned = false;
4611	pIemCpu->aMemMappings[iMemMap].pv = pbBuf;
4612	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
4613	pIemCpu->cActiveMappings++;
4614
4615	*ppvMem = pbBuf;
4616	return VINF_SUCCESS;
4617	}
4618
4619
4620	/**
4621	* iemMemMap woker that deals with iemMemPageMap failures.
4622	*/
4623	static VBOXSTRICTRC iemMemBounceBufferMapPhys(PIEMCPU pIemCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
4624	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
4625	{
4626	/*
4627	* Filter out conditions we can handle and the ones which shouldn't happen.
4628	*/
4629	if ( rcMap != VINF_PGM_PHYS_TLB_CATCH_WRITE
4630	&& rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
4631	&& rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
4632	{
4633	AssertReturn(RT_FAILURE_NP(rcMap), VERR_INTERNAL_ERROR_3);
4634	return rcMap;
4635	}
4636	pIemCpu->cPotentialExits++;
4637
4638	/*
4639	* Read in the current memory content if it's a read, execute or partial
4640	* write access.
4641	*/
4642	uint8_t *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
4643	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
4644	{
4645	if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
4646	memset(pbBuf, 0xff, cbMem);
4647	else
4648	{
4649	int rc;
4650	if (!pIemCpu->fByPassHandlers)
4651	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysFirst, pbBuf, cbMem);
4652	else
4653	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf, GCPhysFirst, cbMem);
4654	if (rc != VINF_SUCCESS)
4655	return rc;
4656	}
4657
4658	#ifdef IEM_VERIFICATION_MODE
4659	if ( !pIemCpu->fNoRem
4660	&& (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC)) )
4661	{
4662	/*
4663	* Record the read.
4664	*/
4665	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
4666	if (pEvtRec)
4667	{
4668	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
4669	pEvtRec->u.RamRead.GCPhys = GCPhysFirst;
4670	pEvtRec->u.RamRead.cb = (uint32_t)cbMem;
4671	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4672	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4673	}
4674	}
4675	#endif
4676	}
4677	#ifdef VBOX_STRICT
4678	else
4679	memset(pbBuf, 0xcc, cbMem);
4680	#endif
4681	#ifdef VBOX_STRICT
4682	if (cbMem < sizeof(pIemCpu->aBounceBuffers[iMemMap].ab))
4683	memset(pbBuf + cbMem, 0xaa, sizeof(pIemCpu->aBounceBuffers[iMemMap].ab) - cbMem);
4684	#endif
4685
4686	/*
4687	* Commit the bounce buffer entry.
4688	*/
4689	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
4690	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
4691	pIemCpu->aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
4692	pIemCpu->aMemBbMappings[iMemMap].cbSecond = 0;
4693	pIemCpu->aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
4694	pIemCpu->aMemMappings[iMemMap].pv = pbBuf;
4695	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
4696	pIemCpu->cActiveMappings++;
4697
4698	*ppvMem = pbBuf;
4699	return VINF_SUCCESS;
4700	}
4701
4702
4703
4704	/**
4705	* Maps the specified guest memory for the given kind of access.
4706	*
4707	* This may be using bounce buffering of the memory if it's crossing a page
4708	* boundary or if there is an access handler installed for any of it. Because
4709	* of lock prefix guarantees, we're in for some extra clutter when this
4710	* happens.
4711	*
4712	* This may raise a \#GP, \#SS, \#PF or \#AC.
4713	*
4714	* @returns VBox strict status code.
4715	*
4716	* @param pIemCpu The IEM per CPU data.
4717	* @param ppvMem Where to return the pointer to the mapped
4718	* memory.
4719	* @param cbMem The number of bytes to map. This is usually 1,
4720	* 2, 4, 6, 8, 12, 16, 32 or 512. When used by
4721	* string operations it can be up to a page.
4722	* @param iSegReg The index of the segment register to use for
4723	* this access. The base and limits are checked.
4724	* Use UINT8_MAX to indicate that no segmentation
4725	* is required (for IDT, GDT and LDT accesses).
4726	* @param GCPtrMem The address of the guest memory.
4727	* @param a_fAccess How the memory is being accessed. The
4728	* IEM_ACCESS_TYPE_XXX bit is used to figure out
4729	* how to map the memory, while the
4730	* IEM_ACCESS_WHAT_XXX bit is used when raising
4731	* exceptions.
4732	*/
4733	static VBOXSTRICTRC iemMemMap(PIEMCPU pIemCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
4734	{
4735	/*
4736	* Check the input and figure out which mapping entry to use.
4737	*/
4738	Assert(cbMem <= 32 \|\| cbMem == 512);
4739	Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK)));
4740
4741	unsigned iMemMap = pIemCpu->iNextMapping;
4742	if (iMemMap >= RT_ELEMENTS(pIemCpu->aMemMappings))
4743	{
4744	iMemMap = iemMemMapFindFree(pIemCpu);
4745	AssertReturn(iMemMap < RT_ELEMENTS(pIemCpu->aMemMappings), VERR_INTERNAL_ERROR_3);
4746	}
4747
4748	/*
4749	* Map the memory, checking that we can actually access it. If something
4750	* slightly complicated happens, fall back on bounce buffering.
4751	*/
4752	VBOXSTRICTRC rcStrict = iemMemApplySegment(pIemCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
4753	if (rcStrict != VINF_SUCCESS)
4754	return rcStrict;
4755
4756	if ((GCPtrMem & PAGE_OFFSET_MASK) + cbMem > PAGE_SIZE) /* Crossing a page boundary? */
4757	return iemMemBounceBufferMapCrossPage(pIemCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
4758
4759	RTGCPHYS GCPhysFirst;
4760	rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrMem, fAccess, &GCPhysFirst);
4761	if (rcStrict != VINF_SUCCESS)
4762	return rcStrict;
4763
4764	void *pvMem;
4765	rcStrict = iemMemPageMap(pIemCpu, GCPhysFirst, fAccess, &pvMem);
4766	if (rcStrict != VINF_SUCCESS)
4767	return iemMemBounceBufferMapPhys(pIemCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
4768
4769	/*
4770	* Fill in the mapping table entry.
4771	*/
4772	pIemCpu->aMemMappings[iMemMap].pv = pvMem;
4773	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess;
4774	pIemCpu->iNextMapping = iMemMap + 1;
4775	pIemCpu->cActiveMappings++;
4776
4777	*ppvMem = pvMem;
4778	return VINF_SUCCESS;
4779	}
4780
4781
4782	/**
4783	* Commits the guest memory if bounce buffered and unmaps it.
4784	*
4785	* @returns Strict VBox status code.
4786	* @param pIemCpu The IEM per CPU data.
4787	* @param pvMem The mapping.
4788	* @param fAccess The kind of access.
4789	*/
4790	static VBOXSTRICTRC iemMemCommitAndUnmap(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess)
4791	{
4792	int iMemMap = iemMapLookup(pIemCpu, pvMem, fAccess);
4793	AssertReturn(iMemMap >= 0, iMemMap);
4794
4795	/*
4796	* If it's bounce buffered, we need to write back the buffer.
4797	*/
4798	if ( (pIemCpu->aMemMappings[iMemMap].fAccess & (IEM_ACCESS_BOUNCE_BUFFERED \| IEM_ACCESS_TYPE_WRITE))
4799	== (IEM_ACCESS_BOUNCE_BUFFERED \| IEM_ACCESS_TYPE_WRITE))
4800	return iemMemBounceBufferCommitAndUnmap(pIemCpu, iMemMap);
4801
4802	/* Free the entry. */
4803	pIemCpu->aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
4804	Assert(pIemCpu->cActiveMappings != 0);
4805	pIemCpu->cActiveMappings--;
4806	return VINF_SUCCESS;
4807	}
4808
4809
4810	/**
4811	* Fetches a data byte.
4812	*
4813	* @returns Strict VBox status code.
4814	* @param pIemCpu The IEM per CPU data.
4815	* @param pu8Dst Where to return the byte.
4816	* @param iSegReg The index of the segment register to use for
4817	* this access. The base and limits are checked.
4818	* @param GCPtrMem The address of the guest memory.
4819	*/
4820	static VBOXSTRICTRC iemMemFetchDataU8(PIEMCPU pIemCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4821	{
4822	/* The lazy approach for now... */
4823	uint8_t const *pu8Src;
4824	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu8Src, sizeof(pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4825	if (rc == VINF_SUCCESS)
4826	{
4827	pu8Dst = pu8Src;
4828	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
4829	}
4830	return rc;
4831	}
4832
4833
4834	/**
4835	* Fetches a data word.
4836	*
4837	* @returns Strict VBox status code.
4838	* @param pIemCpu The IEM per CPU data.
4839	* @param pu16Dst Where to return the word.
4840	* @param iSegReg The index of the segment register to use for
4841	* this access. The base and limits are checked.
4842	* @param GCPtrMem The address of the guest memory.
4843	*/
4844	static VBOXSTRICTRC iemMemFetchDataU16(PIEMCPU pIemCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4845	{
4846	/* The lazy approach for now... */
4847	uint16_t const *pu16Src;
4848	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4849	if (rc == VINF_SUCCESS)
4850	{
4851	pu16Dst = pu16Src;
4852	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
4853	}
4854	return rc;
4855	}
4856
4857
4858	/**
4859	* Fetches a data dword.
4860	*
4861	* @returns Strict VBox status code.
4862	* @param pIemCpu The IEM per CPU data.
4863	* @param pu32Dst Where to return the dword.
4864	* @param iSegReg The index of the segment register to use for
4865	* this access. The base and limits are checked.
4866	* @param GCPtrMem The address of the guest memory.
4867	*/
4868	static VBOXSTRICTRC iemMemFetchDataU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4869	{
4870	/* The lazy approach for now... */
4871	uint32_t const *pu32Src;
4872	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4873	if (rc == VINF_SUCCESS)
4874	{
4875	pu32Dst = pu32Src;
4876	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
4877	}
4878	return rc;
4879	}
4880
4881
4882	#ifdef SOME_UNUSED_FUNCTION
4883	/**
4884	* Fetches a data dword and sign extends it to a qword.
4885	*
4886	* @returns Strict VBox status code.
4887	* @param pIemCpu The IEM per CPU data.
4888	* @param pu64Dst Where to return the sign extended value.
4889	* @param iSegReg The index of the segment register to use for
4890	* this access. The base and limits are checked.
4891	* @param GCPtrMem The address of the guest memory.
4892	*/
4893	static VBOXSTRICTRC iemMemFetchDataS32SxU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4894	{
4895	/* The lazy approach for now... */
4896	int32_t const *pi32Src;
4897	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pi32Src, sizeof(pi32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4898	if (rc == VINF_SUCCESS)
4899	{
4900	pu64Dst = pi32Src;
4901	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
4902	}
4903	#ifdef __GNUC__ /* warning: GCC may be a royal pain */
4904	else
4905	*pu64Dst = 0;
4906	#endif
4907	return rc;
4908	}
4909	#endif
4910
4911
4912	/**
4913	* Fetches a data qword.
4914	*
4915	* @returns Strict VBox status code.
4916	* @param pIemCpu The IEM per CPU data.
4917	* @param pu64Dst Where to return the qword.
4918	* @param iSegReg The index of the segment register to use for
4919	* this access. The base and limits are checked.
4920	* @param GCPtrMem The address of the guest memory.
4921	*/
4922	static VBOXSTRICTRC iemMemFetchDataU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4923	{
4924	/* The lazy approach for now... */
4925	uint64_t const *pu64Src;
4926	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4927	if (rc == VINF_SUCCESS)
4928	{
4929	pu64Dst = pu64Src;
4930	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
4931	}
4932	return rc;
4933	}
4934
4935
4936	/**
4937	* Fetches a data tword.
4938	*
4939	* @returns Strict VBox status code.
4940	* @param pIemCpu The IEM per CPU data.
4941	* @param pr80Dst Where to return the tword.
4942	* @param iSegReg The index of the segment register to use for
4943	* this access. The base and limits are checked.
4944	* @param GCPtrMem The address of the guest memory.
4945	*/
4946	static VBOXSTRICTRC iemMemFetchDataR80(PIEMCPU pIemCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4947	{
4948	/* The lazy approach for now... */
4949	PCRTFLOAT80U pr80Src;
4950	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pr80Src, sizeof(pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4951	if (rc == VINF_SUCCESS)
4952	{
4953	pr80Dst = pr80Src;
4954	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
4955	}
4956	return rc;
4957	}
4958
4959
4960	/**
4961	* Fetches a descriptor register (lgdt, lidt).
4962	*
4963	* @returns Strict VBox status code.
4964	* @param pIemCpu The IEM per CPU data.
4965	* @param pcbLimit Where to return the limit.
4966	* @param pGCPTrBase Where to return the base.
4967	* @param iSegReg The index of the segment register to use for
4968	* this access. The base and limits are checked.
4969	* @param GCPtrMem The address of the guest memory.
4970	* @param enmOpSize The effective operand size.
4971	*/
4972	static VBOXSTRICTRC iemMemFetchDataXdtr(PIEMCPU pIemCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase,
4973	uint8_t iSegReg, RTGCPTR GCPtrMem, IEMMODE enmOpSize)
4974	{
4975	uint8_t const *pu8Src;
4976	VBOXSTRICTRC rcStrict = iemMemMap(pIemCpu,
4977	(void **)&pu8Src,
4978	enmOpSize == IEMMODE_64BIT
4979	? 2 + 8
4980	: enmOpSize == IEMMODE_32BIT
4981	? 2 + 4
4982	: 2 + 3,
4983	iSegReg,
4984	GCPtrMem,
4985	IEM_ACCESS_DATA_R);
4986	if (rcStrict == VINF_SUCCESS)
4987	{
4988	*pcbLimit = RT_MAKE_U16(pu8Src[0], pu8Src[1]);
4989	switch (enmOpSize)
4990	{
4991	case IEMMODE_16BIT:
4992	*pGCPtrBase = RT_MAKE_U32_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], 0);
4993	break;
4994	case IEMMODE_32BIT:
4995	*pGCPtrBase = RT_MAKE_U32_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], pu8Src[5]);
4996	break;
4997	case IEMMODE_64BIT:
4998	*pGCPtrBase = RT_MAKE_U64_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], pu8Src[5],
4999	pu8Src[6], pu8Src[7], pu8Src[8], pu8Src[9]);
5000	break;
5001
5002	IEM_NOT_REACHED_DEFAULT_CASE_RET();
5003	}
5004	rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
5005	}
5006	return rcStrict;
5007	}
5008
5009
5010
5011	/**
5012	* Stores a data byte.
5013	*
5014	* @returns Strict VBox status code.
5015	* @param pIemCpu The IEM per CPU data.
5016	* @param iSegReg The index of the segment register to use for
5017	* this access. The base and limits are checked.
5018	* @param GCPtrMem The address of the guest memory.
5019	* @param u8Value The value to store.
5020	*/
5021	static VBOXSTRICTRC iemMemStoreDataU8(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
5022	{
5023	/* The lazy approach for now... */
5024	uint8_t *pu8Dst;
5025	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu8Dst, sizeof(pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
5026	if (rc == VINF_SUCCESS)
5027	{
5028	*pu8Dst = u8Value;
5029	rc = iemMemCommitAndUnmap(pIemCpu, pu8Dst, IEM_ACCESS_DATA_W);
5030	}
5031	return rc;
5032	}
5033
5034
5035	/**
5036	* Stores a data word.
5037	*
5038	* @returns Strict VBox status code.
5039	* @param pIemCpu The IEM per CPU data.
5040	* @param iSegReg The index of the segment register to use for
5041	* this access. The base and limits are checked.
5042	* @param GCPtrMem The address of the guest memory.
5043	* @param u16Value The value to store.
5044	*/
5045	static VBOXSTRICTRC iemMemStoreDataU16(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
5046	{
5047	/* The lazy approach for now... */
5048	uint16_t *pu16Dst;
5049	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
5050	if (rc == VINF_SUCCESS)
5051	{
5052	*pu16Dst = u16Value;
5053	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_DATA_W);
5054	}
5055	return rc;
5056	}
5057
5058
5059	/**
5060	* Stores a data dword.
5061	*
5062	* @returns Strict VBox status code.
5063	* @param pIemCpu The IEM per CPU data.
5064	* @param iSegReg The index of the segment register to use for
5065	* this access. The base and limits are checked.
5066	* @param GCPtrMem The address of the guest memory.
5067	* @param u32Value The value to store.
5068	*/
5069	static VBOXSTRICTRC iemMemStoreDataU32(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
5070	{
5071	/* The lazy approach for now... */
5072	uint32_t *pu32Dst;
5073	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
5074	if (rc == VINF_SUCCESS)
5075	{
5076	*pu32Dst = u32Value;
5077	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_DATA_W);
5078	}
5079	return rc;
5080	}
5081
5082
5083	/**
5084	* Stores a data qword.
5085	*
5086	* @returns Strict VBox status code.
5087	* @param pIemCpu The IEM per CPU data.
5088	* @param iSegReg The index of the segment register to use for
5089	* this access. The base and limits are checked.
5090	* @param GCPtrMem The address of the guest memory.
5091	* @param u64Value The value to store.
5092	*/
5093	static VBOXSTRICTRC iemMemStoreDataU64(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
5094	{
5095	/* The lazy approach for now... */
5096	uint64_t *pu64Dst;
5097	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
5098	if (rc == VINF_SUCCESS)
5099	{
5100	*pu64Dst = u64Value;
5101	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_DATA_W);
5102	}
5103	return rc;
5104	}
5105
5106
5107	/**
5108	* Pushes a word onto the stack.
5109	*
5110	* @returns Strict VBox status code.
5111	* @param pIemCpu The IEM per CPU data.
5112	* @param u16Value The value to push.
5113	*/
5114	static VBOXSTRICTRC iemMemStackPushU16(PIEMCPU pIemCpu, uint16_t u16Value)
5115	{
5116	/* Increment the stack pointer. */
5117	uint64_t uNewRsp;
5118	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5119	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, 2, &uNewRsp);
5120
5121	/* Write the word the lazy way. */
5122	uint16_t *pu16Dst;
5123	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5124	if (rc == VINF_SUCCESS)
5125	{
5126	*pu16Dst = u16Value;
5127	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_STACK_W);
5128	}
5129
5130	/* Commit the new RSP value unless we an access handler made trouble. */
5131	if (rc == VINF_SUCCESS)
5132	pCtx->rsp = uNewRsp;
5133
5134	return rc;
5135	}
5136
5137
5138	/**
5139	* Pushes a dword onto the stack.
5140	*
5141	* @returns Strict VBox status code.
5142	* @param pIemCpu The IEM per CPU data.
5143	* @param u32Value The value to push.
5144	*/
5145	static VBOXSTRICTRC iemMemStackPushU32(PIEMCPU pIemCpu, uint32_t u32Value)
5146	{
5147	/* Increment the stack pointer. */
5148	uint64_t uNewRsp;
5149	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5150	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, 4, &uNewRsp);
5151
5152	/* Write the word the lazy way. */
5153	uint32_t *pu32Dst;
5154	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5155	if (rc == VINF_SUCCESS)
5156	{
5157	*pu32Dst = u32Value;
5158	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_STACK_W);
5159	}
5160
5161	/* Commit the new RSP value unless we an access handler made trouble. */
5162	if (rc == VINF_SUCCESS)
5163	pCtx->rsp = uNewRsp;
5164
5165	return rc;
5166	}
5167
5168
5169	/**
5170	* Pushes a qword onto the stack.
5171	*
5172	* @returns Strict VBox status code.
5173	* @param pIemCpu The IEM per CPU data.
5174	* @param u64Value The value to push.
5175	*/
5176	static VBOXSTRICTRC iemMemStackPushU64(PIEMCPU pIemCpu, uint64_t u64Value)
5177	{
5178	/* Increment the stack pointer. */
5179	uint64_t uNewRsp;
5180	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5181	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, 8, &uNewRsp);
5182
5183	/* Write the word the lazy way. */
5184	uint64_t *pu64Dst;
5185	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5186	if (rc == VINF_SUCCESS)
5187	{
5188	*pu64Dst = u64Value;
5189	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_STACK_W);
5190	}
5191
5192	/* Commit the new RSP value unless we an access handler made trouble. */
5193	if (rc == VINF_SUCCESS)
5194	pCtx->rsp = uNewRsp;
5195
5196	return rc;
5197	}
5198
5199
5200	/**
5201	* Pops a word from the stack.
5202	*
5203	* @returns Strict VBox status code.
5204	* @param pIemCpu The IEM per CPU data.
5205	* @param pu16Value Where to store the popped value.
5206	*/
5207	static VBOXSTRICTRC iemMemStackPopU16(PIEMCPU pIemCpu, uint16_t *pu16Value)
5208	{
5209	/* Increment the stack pointer. */
5210	uint64_t uNewRsp;
5211	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5212	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, 2, &uNewRsp);
5213
5214	/* Write the word the lazy way. */
5215	uint16_t const *pu16Src;
5216	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5217	if (rc == VINF_SUCCESS)
5218	{
5219	pu16Value = pu16Src;
5220	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
5221
5222	/* Commit the new RSP value. */
5223	if (rc == VINF_SUCCESS)
5224	pCtx->rsp = uNewRsp;
5225	}
5226
5227	return rc;
5228	}
5229
5230
5231	/**
5232	* Pops a dword from the stack.
5233	*
5234	* @returns Strict VBox status code.
5235	* @param pIemCpu The IEM per CPU data.
5236	* @param pu32Value Where to store the popped value.
5237	*/
5238	static VBOXSTRICTRC iemMemStackPopU32(PIEMCPU pIemCpu, uint32_t *pu32Value)
5239	{
5240	/* Increment the stack pointer. */
5241	uint64_t uNewRsp;
5242	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5243	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, 4, &uNewRsp);
5244
5245	/* Write the word the lazy way. */
5246	uint32_t const *pu32Src;
5247	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5248	if (rc == VINF_SUCCESS)
5249	{
5250	pu32Value = pu32Src;
5251	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
5252
5253	/* Commit the new RSP value. */
5254	if (rc == VINF_SUCCESS)
5255	pCtx->rsp = uNewRsp;
5256	}
5257
5258	return rc;
5259	}
5260
5261
5262	/**
5263	* Pops a qword from the stack.
5264	*
5265	* @returns Strict VBox status code.
5266	* @param pIemCpu The IEM per CPU data.
5267	* @param pu64Value Where to store the popped value.
5268	*/
5269	static VBOXSTRICTRC iemMemStackPopU64(PIEMCPU pIemCpu, uint64_t *pu64Value)
5270	{
5271	/* Increment the stack pointer. */
5272	uint64_t uNewRsp;
5273	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5274	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, 8, &uNewRsp);
5275
5276	/* Write the word the lazy way. */
5277	uint64_t const *pu64Src;
5278	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5279	if (rc == VINF_SUCCESS)
5280	{
5281	pu64Value = pu64Src;
5282	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
5283
5284	/* Commit the new RSP value. */
5285	if (rc == VINF_SUCCESS)
5286	pCtx->rsp = uNewRsp;
5287	}
5288
5289	return rc;
5290	}
5291
5292
5293	/**
5294	* Pushes a word onto the stack, using a temporary stack pointer.
5295	*
5296	* @returns Strict VBox status code.
5297	* @param pIemCpu The IEM per CPU data.
5298	* @param u16Value The value to push.
5299	* @param pTmpRsp Pointer to the temporary stack pointer.
5300	*/
5301	static VBOXSTRICTRC iemMemStackPushU16Ex(PIEMCPU pIemCpu, uint16_t u16Value, PRTUINT64U pTmpRsp)
5302	{
5303	/* Increment the stack pointer. */
5304	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5305	RTUINT64U NewRsp = *pTmpRsp;
5306	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(&NewRsp, 2, pCtx);
5307
5308	/* Write the word the lazy way. */
5309	uint16_t *pu16Dst;
5310	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5311	if (rc == VINF_SUCCESS)
5312	{
5313	*pu16Dst = u16Value;
5314	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_STACK_W);
5315	}
5316
5317	/* Commit the new RSP value unless we an access handler made trouble. */
5318	if (rc == VINF_SUCCESS)
5319	*pTmpRsp = NewRsp;
5320
5321	return rc;
5322	}
5323
5324
5325	/**
5326	* Pushes a dword onto the stack, using a temporary stack pointer.
5327	*
5328	* @returns Strict VBox status code.
5329	* @param pIemCpu The IEM per CPU data.
5330	* @param u32Value The value to push.
5331	* @param pTmpRsp Pointer to the temporary stack pointer.
5332	*/
5333	static VBOXSTRICTRC iemMemStackPushU32Ex(PIEMCPU pIemCpu, uint32_t u32Value, PRTUINT64U pTmpRsp)
5334	{
5335	/* Increment the stack pointer. */
5336	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5337	RTUINT64U NewRsp = *pTmpRsp;
5338	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(&NewRsp, 4, pCtx);
5339
5340	/* Write the word the lazy way. */
5341	uint32_t *pu32Dst;
5342	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5343	if (rc == VINF_SUCCESS)
5344	{
5345	*pu32Dst = u32Value;
5346	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_STACK_W);
5347	}
5348
5349	/* Commit the new RSP value unless we an access handler made trouble. */
5350	if (rc == VINF_SUCCESS)
5351	*pTmpRsp = NewRsp;
5352
5353	return rc;
5354	}
5355
5356
5357	#ifdef SOME_UNUSED_FUNCTION
5358	/**
5359	* Pushes a dword onto the stack, using a temporary stack pointer.
5360	*
5361	* @returns Strict VBox status code.
5362	* @param pIemCpu The IEM per CPU data.
5363	* @param u64Value The value to push.
5364	* @param pTmpRsp Pointer to the temporary stack pointer.
5365	*/
5366	static VBOXSTRICTRC iemMemStackPushU64Ex(PIEMCPU pIemCpu, uint64_t u64Value, PRTUINT64U pTmpRsp)
5367	{
5368	/* Increment the stack pointer. */
5369	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5370	RTUINT64U NewRsp = *pTmpRsp;
5371	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(&NewRsp, 8, pCtx);
5372
5373	/* Write the word the lazy way. */
5374	uint64_t *pu64Dst;
5375	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5376	if (rc == VINF_SUCCESS)
5377	{
5378	*pu64Dst = u64Value;
5379	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_STACK_W);
5380	}
5381
5382	/* Commit the new RSP value unless we an access handler made trouble. */
5383	if (rc == VINF_SUCCESS)
5384	*pTmpRsp = NewRsp;
5385
5386	return rc;
5387	}
5388	#endif
5389
5390
5391	/**
5392	* Pops a word from the stack, using a temporary stack pointer.
5393	*
5394	* @returns Strict VBox status code.
5395	* @param pIemCpu The IEM per CPU data.
5396	* @param pu16Value Where to store the popped value.
5397	* @param pTmpRsp Pointer to the temporary stack pointer.
5398	*/
5399	static VBOXSTRICTRC iemMemStackPopU16Ex(PIEMCPU pIemCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp)
5400	{
5401	/* Increment the stack pointer. */
5402	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5403	RTUINT64U NewRsp = *pTmpRsp;
5404	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 2, pCtx);
5405
5406	/* Write the word the lazy way. */
5407	uint16_t const *pu16Src;
5408	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5409	if (rc == VINF_SUCCESS)
5410	{
5411	pu16Value = pu16Src;
5412	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
5413
5414	/* Commit the new RSP value. */
5415	if (rc == VINF_SUCCESS)
5416	*pTmpRsp = NewRsp;
5417	}
5418
5419	return rc;
5420	}
5421
5422
5423	/**
5424	* Pops a dword from the stack, using a temporary stack pointer.
5425	*
5426	* @returns Strict VBox status code.
5427	* @param pIemCpu The IEM per CPU data.
5428	* @param pu32Value Where to store the popped value.
5429	* @param pTmpRsp Pointer to the temporary stack pointer.
5430	*/
5431	static VBOXSTRICTRC iemMemStackPopU32Ex(PIEMCPU pIemCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp)
5432	{
5433	/* Increment the stack pointer. */
5434	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5435	RTUINT64U NewRsp = *pTmpRsp;
5436	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 4, pCtx);
5437
5438	/* Write the word the lazy way. */
5439	uint32_t const *pu32Src;
5440	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5441	if (rc == VINF_SUCCESS)
5442	{
5443	pu32Value = pu32Src;
5444	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
5445
5446	/* Commit the new RSP value. */
5447	if (rc == VINF_SUCCESS)
5448	*pTmpRsp = NewRsp;
5449	}
5450
5451	return rc;
5452	}
5453
5454
5455	/**
5456	* Pops a qword from the stack, using a temporary stack pointer.
5457	*
5458	* @returns Strict VBox status code.
5459	* @param pIemCpu The IEM per CPU data.
5460	* @param pu64Value Where to store the popped value.
5461	* @param pTmpRsp Pointer to the temporary stack pointer.
5462	*/
5463	static VBOXSTRICTRC iemMemStackPopU64Ex(PIEMCPU pIemCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp)
5464	{
5465	/* Increment the stack pointer. */
5466	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5467	RTUINT64U NewRsp = *pTmpRsp;
5468	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 8, pCtx);
5469
5470	/* Write the word the lazy way. */
5471	uint64_t const *pu64Src;
5472	VBOXSTRICTRC rcStrict = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5473	if (rcStrict == VINF_SUCCESS)
5474	{
5475	pu64Value = pu64Src;
5476	rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
5477
5478	/* Commit the new RSP value. */
5479	if (rcStrict == VINF_SUCCESS)
5480	*pTmpRsp = NewRsp;
5481	}
5482
5483	return rcStrict;
5484	}
5485
5486
5487	/**
5488	* Begin a special stack push (used by interrupt, exceptions and such).
5489	*
5490	* This will raise #SS or #PF if appropriate.
5491	*
5492	* @returns Strict VBox status code.
5493	* @param pIemCpu The IEM per CPU data.
5494	* @param cbMem The number of bytes to push onto the stack.
5495	* @param ppvMem Where to return the pointer to the stack memory.
5496	* As with the other memory functions this could be
5497	* direct access or bounce buffered access, so
5498	* don't commit register until the commit call
5499	* succeeds.
5500	* @param puNewRsp Where to return the new RSP value. This must be
5501	* passed unchanged to
5502	* iemMemStackPushCommitSpecial().
5503	*/
5504	static VBOXSTRICTRC iemMemStackPushBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void *ppvMem, uint64_t puNewRsp)
5505	{
5506	Assert(cbMem < UINT8_MAX);
5507	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5508	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, (uint8_t)cbMem, puNewRsp);
5509	return iemMemMap(pIemCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5510	}
5511
5512
5513	/**
5514	* Commits a special stack push (started by iemMemStackPushBeginSpecial).
5515	*
5516	* This will update the rSP.
5517	*
5518	* @returns Strict VBox status code.
5519	* @param pIemCpu The IEM per CPU data.
5520	* @param pvMem The pointer returned by
5521	* iemMemStackPushBeginSpecial().
5522	* @param uNewRsp The new RSP value returned by
5523	* iemMemStackPushBeginSpecial().
5524	*/
5525	static VBOXSTRICTRC iemMemStackPushCommitSpecial(PIEMCPU pIemCpu, void *pvMem, uint64_t uNewRsp)
5526	{
5527	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pIemCpu, pvMem, IEM_ACCESS_STACK_W);
5528	if (rcStrict == VINF_SUCCESS)
5529	pIemCpu->CTX_SUFF(pCtx)->rsp = uNewRsp;
5530	return rcStrict;
5531	}
5532
5533
5534	/**
5535	* Begin a special stack pop (used by iret, retf and such).
5536	*
5537	* This will raise \#SS or \#PF if appropriate.
5538	*
5539	* @returns Strict VBox status code.
5540	* @param pIemCpu The IEM per CPU data.
5541	* @param cbMem The number of bytes to push onto the stack.
5542	* @param ppvMem Where to return the pointer to the stack memory.
5543	* @param puNewRsp Where to return the new RSP value. This must be
5544	* passed unchanged to
5545	* iemMemStackPopCommitSpecial() or applied
5546	* manually if iemMemStackPopDoneSpecial() is used.
5547	*/
5548	static VBOXSTRICTRC iemMemStackPopBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp)
5549	{
5550	Assert(cbMem < UINT8_MAX);
5551	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5552	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, (uint8_t)cbMem, puNewRsp);
5553	return iemMemMap(pIemCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5554	}
5555
5556
5557	/**
5558	* Continue a special stack pop (used by iret and retf).
5559	*
5560	* This will raise \#SS or \#PF if appropriate.
5561	*
5562	* @returns Strict VBox status code.
5563	* @param pIemCpu The IEM per CPU data.
5564	* @param cbMem The number of bytes to push onto the stack.
5565	* @param ppvMem Where to return the pointer to the stack memory.
5566	* @param puNewRsp Where to return the new RSP value. This must be
5567	* passed unchanged to
5568	* iemMemStackPopCommitSpecial() or applied
5569	* manually if iemMemStackPopDoneSpecial() is used.
5570	*/
5571	static VBOXSTRICTRC iemMemStackPopContinueSpecial(PIEMCPU pIemCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp)
5572	{
5573	Assert(cbMem < UINT8_MAX);
5574	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5575	RTUINT64U NewRsp;
5576	NewRsp.u = *puNewRsp;
5577	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 8, pCtx);
5578	*puNewRsp = NewRsp.u;
5579	return iemMemMap(pIemCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5580	}
5581
5582
5583	/**
5584	* Commits a special stack pop (started by iemMemStackPopBeginSpecial).
5585	*
5586	* This will update the rSP.
5587	*
5588	* @returns Strict VBox status code.
5589	* @param pIemCpu The IEM per CPU data.
5590	* @param pvMem The pointer returned by
5591	* iemMemStackPopBeginSpecial().
5592	* @param uNewRsp The new RSP value returned by
5593	* iemMemStackPopBeginSpecial().
5594	*/
5595	static VBOXSTRICTRC iemMemStackPopCommitSpecial(PIEMCPU pIemCpu, void const *pvMem, uint64_t uNewRsp)
5596	{
5597	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
5598	if (rcStrict == VINF_SUCCESS)
5599	pIemCpu->CTX_SUFF(pCtx)->rsp = uNewRsp;
5600	return rcStrict;
5601	}
5602
5603
5604	/**
5605	* Done with a special stack pop (started by iemMemStackPopBeginSpecial or
5606	* iemMemStackPopContinueSpecial).
5607	*
5608	* The caller will manually commit the rSP.
5609	*
5610	* @returns Strict VBox status code.
5611	* @param pIemCpu The IEM per CPU data.
5612	* @param pvMem The pointer returned by
5613	* iemMemStackPopBeginSpecial() or
5614	* iemMemStackPopContinueSpecial().
5615	*/
5616	static VBOXSTRICTRC iemMemStackPopDoneSpecial(PIEMCPU pIemCpu, void const *pvMem)
5617	{
5618	return iemMemCommitAndUnmap(pIemCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
5619	}
5620
5621
5622	/**
5623	* Fetches a system table dword.
5624	*
5625	* @returns Strict VBox status code.
5626	* @param pIemCpu The IEM per CPU data.
5627	* @param pu32Dst Where to return the dword.
5628	* @param iSegReg The index of the segment register to use for
5629	* this access. The base and limits are checked.
5630	* @param GCPtrMem The address of the guest memory.
5631	*/
5632	static VBOXSTRICTRC iemMemFetchSysU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
5633	{
5634	/* The lazy approach for now... */
5635	uint32_t const *pu32Src;
5636	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
5637	if (rc == VINF_SUCCESS)
5638	{
5639	pu32Dst = pu32Src;
5640	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
5641	}
5642	return rc;
5643	}
5644
5645
5646	/**
5647	* Fetches a system table qword.
5648	*
5649	* @returns Strict VBox status code.
5650	* @param pIemCpu The IEM per CPU data.
5651	* @param pu64Dst Where to return the qword.
5652	* @param iSegReg The index of the segment register to use for
5653	* this access. The base and limits are checked.
5654	* @param GCPtrMem The address of the guest memory.
5655	*/
5656	static VBOXSTRICTRC iemMemFetchSysU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
5657	{
5658	/* The lazy approach for now... */
5659	uint64_t const *pu64Src;
5660	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
5661	if (rc == VINF_SUCCESS)
5662	{
5663	pu64Dst = pu64Src;
5664	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
5665	}
5666	return rc;
5667	}
5668
5669
5670	/**
5671	* Fetches a descriptor table entry.
5672	*
5673	* @returns Strict VBox status code.
5674	* @param pIemCpu The IEM per CPU.
5675	* @param pDesc Where to return the descriptor table entry.
5676	* @param uSel The selector which table entry to fetch.
5677	*/
5678	static VBOXSTRICTRC iemMemFetchSelDesc(PIEMCPU pIemCpu, PIEMSELDESC pDesc, uint16_t uSel)
5679	{
5680	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5681
5682	/** @todo did the 286 require all 8 bytes to be accessible? */
5683	/*
5684	* Get the selector table base and check bounds.
5685	*/
5686	RTGCPTR GCPtrBase;
5687	if (uSel & X86_SEL_LDT)
5688	{
5689	if ( !pCtx->ldtrHid.Attr.n.u1Present
5690	\|\| (uSel \| 0x7U) > pCtx->ldtrHid.u32Limit )
5691	{
5692	Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
5693	uSel, pCtx->ldtrHid.u32Limit, pCtx->ldtr));
5694	/** @todo is this the right exception? */
5695	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, uSel);
5696	}
5697
5698	Assert(pCtx->ldtrHid.Attr.n.u1Present);
5699	GCPtrBase = pCtx->ldtrHid.u64Base;
5700	}
5701	else
5702	{
5703	if ((uSel \| 0x7U) > pCtx->gdtr.cbGdt)
5704	{
5705	Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pCtx->gdtr.cbGdt));
5706	/** @todo is this the right exception? */
5707	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, uSel);
5708	}
5709	GCPtrBase = pCtx->gdtr.pGdt;
5710	}
5711
5712	/*
5713	* Read the legacy descriptor and maybe the long mode extensions if
5714	* required.
5715	*/
5716	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pIemCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
5717	if (rcStrict == VINF_SUCCESS)
5718	{
5719	if ( !IEM_IS_LONG_MODE(pIemCpu)
5720	\|\| pDesc->Legacy.Gen.u1DescType)
5721	pDesc->Long.au64[1] = 0;
5722	else if ((uint32_t)(uSel & X86_SEL_MASK) + 15 < (uSel & X86_SEL_LDT ? pCtx->ldtrHid.u32Limit : pCtx->gdtr.cbGdt))
5723	rcStrict = iemMemFetchSysU64(pIemCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
5724	else
5725	{
5726	Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
5727	/** @todo is this the right exception? */
5728	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, uSel);
5729	}
5730	}
5731	return rcStrict;
5732	}
5733
5734
5735	/**
5736	* Fakes a long mode stack selector for SS = 0.
5737	*
5738	* @param pDescSs Where to return the fake stack descriptor.
5739	* @param uDpl The DPL we want.
5740	*/
5741	static void iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
5742	{
5743	pDescSs->Long.au64[0] = 0;
5744	pDescSs->Long.au64[1] = 0;
5745	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
5746	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
5747	pDescSs->Long.Gen.u2Dpl = uDpl;
5748	pDescSs->Long.Gen.u1Present = 1;
5749	pDescSs->Long.Gen.u1Long = 1;
5750	}
5751
5752
5753	/**
5754	* Marks the selector descriptor as accessed (only non-system descriptors).
5755	*
5756	* This function ASSUMES that iemMemFetchSelDesc has be called previously and
5757	* will therefore skip the limit checks.
5758	*
5759	* @returns Strict VBox status code.
5760	* @param pIemCpu The IEM per CPU.
5761	* @param uSel The selector.
5762	*/
5763	static VBOXSTRICTRC iemMemMarkSelDescAccessed(PIEMCPU pIemCpu, uint16_t uSel)
5764	{
5765	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5766
5767	/*
5768	* Get the selector table base and calculate the entry address.
5769	*/
5770	RTGCPTR GCPtr = uSel & X86_SEL_LDT
5771	? pCtx->ldtrHid.u64Base
5772	: pCtx->gdtr.pGdt;
5773	GCPtr += uSel & X86_SEL_MASK;
5774
5775	/*
5776	* ASMAtomicBitSet will assert if the address is misaligned, so do some
5777	* ugly stuff to avoid this. This will make sure it's an atomic access
5778	* as well more or less remove any question about 8-bit or 32-bit accesss.
5779	*/
5780	VBOXSTRICTRC rcStrict;
5781	uint32_t volatile *pu32;
5782	if ((GCPtr & 3) == 0)
5783	{
5784	/* The normal case, map the 32-bit bits around the accessed bit (40). */
5785	GCPtr += 2 + 2;
5786	rcStrict = iemMemMap(pIemCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
5787	if (rcStrict != VINF_SUCCESS)
5788	return rcStrict;
5789	ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
5790	}
5791	else
5792	{
5793	/* The misaligned GDT/LDT case, map the whole thing. */
5794	rcStrict = iemMemMap(pIemCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
5795	if (rcStrict != VINF_SUCCESS)
5796	return rcStrict;
5797	switch ((uintptr_t)pu32 & 3)
5798	{
5799	case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
5800	case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
5801	case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
5802	case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
5803	}
5804	}
5805
5806	return iemMemCommitAndUnmap(pIemCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
5807	}
5808
5809	/** @} */
5810
5811
5812	/*
5813	* Include the C/C++ implementation of instruction.
5814	*/
5815	#include "IEMAllCImpl.cpp.h"
5816
5817
5818
5819	/** @name "Microcode" macros.
5820	*
5821	* The idea is that we should be able to use the same code to interpret
5822	* instructions as well as recompiler instructions. Thus this obfuscation.
5823	*
5824	* @{
5825	*/
5826	#define IEM_MC_BEGIN(a_cArgs, a_cLocals) {
5827	#define IEM_MC_END() }
5828	#define IEM_MC_PAUSE() do {} while (0)
5829	#define IEM_MC_CONTINUE() do {} while (0)
5830
5831	/** Internal macro. */
5832	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
5833	do \
5834	{ \
5835	VBOXSTRICTRC rcStrict2 = a_Expr; \
5836	if (rcStrict2 != VINF_SUCCESS) \
5837	return rcStrict2; \
5838	} while (0)
5839
5840	#define IEM_MC_ADVANCE_RIP() iemRegUpdateRip(pIemCpu)
5841	#define IEM_MC_REL_JMP_S8(a_i8) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS8(pIemCpu, a_i8))
5842	#define IEM_MC_REL_JMP_S16(a_i16) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS16(pIemCpu, a_i16))
5843	#define IEM_MC_REL_JMP_S32(a_i32) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS32(pIemCpu, a_i32))
5844	#define IEM_MC_SET_RIP_U16(a_u16NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u16NewIP)))
5845	#define IEM_MC_SET_RIP_U32(a_u32NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u32NewIP)))
5846	#define IEM_MC_SET_RIP_U64(a_u64NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u64NewIP)))
5847
5848	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pIemCpu)
5849	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
5850	do { \
5851	if ((pIemCpu)->CTX_SUFF(pCtx)->cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
5852	return iemRaiseDeviceNotAvailable(pIemCpu); \
5853	} while (0)
5854	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
5855	do { \
5856	if ((pIemCpu)->CTX_SUFF(pCtx)->fpu.FSW & X86_FSW_ES) \
5857	return iemRaiseMathFault(pIemCpu); \
5858	} while (0)
5859	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
5860	do { \
5861	if (pIemCpu->uCpl != 0) \
5862	return iemRaiseGeneralProtectionFault0(pIemCpu); \
5863	} while (0)
5864
5865
5866	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
5867	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
5868	#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
5869	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
5870	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
5871	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
5872	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
5873	uint32_t a_Name; \
5874	uint32_t *a_pName = &a_Name
5875	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
5876	do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u = (a_EFlags); Assert((pIemCpu)->CTX_SUFF(pCtx)->eflags.u & X86_EFL_1); } while (0)
5877
5878	#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
5879	#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
5880
5881	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5882	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5883	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5884	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5885	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
5886	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
5887	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
5888	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
5889	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
5890	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
5891	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pIemCpu, (a_iGReg))
5892	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pIemCpu, (a_iGReg))
5893	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pIemCpu, (a_iGReg))
5894	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pIemCpu, (a_iGReg))
5895	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pIemCpu, (a_iGReg))
5896	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pIemCpu, (a_iGReg))
5897	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
5898	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) (a_u16Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
5899	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) (a_u32Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
5900	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) (a_u64Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
5901	#define IEM_MC_FETCH_CR0_U16(a_u16Dst) (a_u16Dst) = (uint16_t)(pIemCpu)->CTX_SUFF(pCtx)->cr0
5902	#define IEM_MC_FETCH_CR0_U32(a_u32Dst) (a_u32Dst) = (uint32_t)(pIemCpu)->CTX_SUFF(pCtx)->cr0
5903	#define IEM_MC_FETCH_CR0_U64(a_u64Dst) (a_u64Dst) = (pIemCpu)->CTX_SUFF(pCtx)->cr0
5904	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = (pIemCpu)->CTX_SUFF(pCtx)->eflags.u
5905	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)(pIemCpu)->CTX_SUFF(pCtx)->eflags.u
5906	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pIemCpu->CTX_SUFF(pCtx)->fpu.FSW
5907	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pIemCpu->CTX_SUFF(pCtx)->fpu.FCW
5908
5909	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8(pIemCpu, (a_iGReg)) = (a_u8Value)
5910	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) = (a_u16Value)
5911	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) = (uint32_t)(a_u32Value) /* clear high bits. */
5912	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) = (a_u64Value)
5913	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
5914	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
5915	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
5916	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
5917	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) &= UINT32_MAX
5918	#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
5919	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
5920	do { pIemCpu->CTX_SUFF(pCtx)->fpu.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
5921
5922	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8(pIemCpu, (a_iGReg))
5923	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t *)iemGRegRef(pIemCpu, (a_iGReg))
5924	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
5925	* Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
5926	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t *)iemGRegRef(pIemCpu, (a_iGReg))
5927	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t *)iemGRegRef(pIemCpu, (a_iGReg))
5928	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &(pIemCpu)->CTX_SUFF(pCtx)->eflags.u
5929
5930	#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u8Value)
5931	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u16Value)
5932	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
5933	do { \
5934	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
5935	*pu32Reg += (a_u32Value); \
5936	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
5937	} while (0)
5938	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u64Value)
5939
5940	#define IEM_MC_SUB_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u8Value)
5941	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u16Value)
5942	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
5943	do { \
5944	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
5945	*pu32Reg -= (a_u32Value); \
5946	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
5947	} while (0)
5948	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u64Value)
5949
5950	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pIemCpu, (a_iGReg)); } while (0)
5951	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pIemCpu, (a_iGReg)); } while (0)
5952	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pIemCpu, (a_iGReg)); } while (0)
5953	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pIemCpu, (a_iGReg)); } while (0)
5954	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
5955	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
5956	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
5957
5958	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
5959	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
5960	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
5961	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
5962
5963	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
5964	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
5965	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
5966
5967	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
5968	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
5969
5970	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
5971	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
5972	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
5973
5974	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
5975	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
5976	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
5977
5978	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
5979
5980	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
5981
5982	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u8Value)
5983	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u16Value)
5984	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
5985	do { \
5986	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
5987	*pu32Reg &= (a_u32Value); \
5988	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
5989	} while (0)
5990	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u64Value)
5991
5992	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u8Value)
5993	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u16Value)
5994	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
5995	do { \
5996	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
5997	*pu32Reg \|= (a_u32Value); \
5998	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
5999	} while (0)
6000	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u64Value)
6001
6002
6003	#define IEM_MC_SET_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u \|= (a_fBit); } while (0)
6004	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u &= ~(a_fBit); } while (0)
6005	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u ^= (a_fBit); } while (0)
6006
6007	#define IEM_MC_CLEAR_FSW_EX() do { (pIemCpu)->CTX_SUFF(pCtx)->fpu.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
6008
6009
6010	#define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
6011	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
6012	#define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
6013	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
6014	#define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
6015	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
6016
6017	#define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
6018	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
6019	#define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
6020	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
6021	#define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
6022	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
6023
6024	#define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
6025	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
6026	#define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
6027	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
6028	#define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
6029	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
6030
6031	#define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6032	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
6033
6034	#define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6035	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
6036	#define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
6037	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
6038
6039	#define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
6040	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_r32Dst).u32, (a_iSeg), (a_GCPtrMem)))
6041	#define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
6042	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_r64Dst).au64[0], (a_iSeg), (a_GCPtrMem)))
6043	#define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
6044	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pIemCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
6045
6046
6047	#define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
6048	do { \
6049	uint8_t u8Tmp; \
6050	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6051	(a_u16Dst) = u8Tmp; \
6052	} while (0)
6053	#define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
6054	do { \
6055	uint8_t u8Tmp; \
6056	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6057	(a_u32Dst) = u8Tmp; \
6058	} while (0)
6059	#define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6060	do { \
6061	uint8_t u8Tmp; \
6062	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6063	(a_u64Dst) = u8Tmp; \
6064	} while (0)
6065	#define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
6066	do { \
6067	uint16_t u16Tmp; \
6068	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
6069	(a_u32Dst) = u16Tmp; \
6070	} while (0)
6071	#define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6072	do { \
6073	uint16_t u16Tmp; \
6074	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
6075	(a_u64Dst) = u16Tmp; \
6076	} while (0)
6077	#define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6078	do { \
6079	uint32_t u32Tmp; \
6080	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
6081	(a_u64Dst) = u32Tmp; \
6082	} while (0)
6083
6084	#define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
6085	do { \
6086	uint8_t u8Tmp; \
6087	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6088	(a_u16Dst) = (int8_t)u8Tmp; \
6089	} while (0)
6090	#define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
6091	do { \
6092	uint8_t u8Tmp; \
6093	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6094	(a_u32Dst) = (int8_t)u8Tmp; \
6095	} while (0)
6096	#define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6097	do { \
6098	uint8_t u8Tmp; \
6099	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6100	(a_u64Dst) = (int8_t)u8Tmp; \
6101	} while (0)
6102	#define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
6103	do { \
6104	uint16_t u16Tmp; \
6105	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
6106	(a_u32Dst) = (int16_t)u16Tmp; \
6107	} while (0)
6108	#define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6109	do { \
6110	uint16_t u16Tmp; \
6111	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
6112	(a_u64Dst) = (int16_t)u16Tmp; \
6113	} while (0)
6114	#define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6115	do { \
6116	uint32_t u32Tmp; \
6117	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
6118	(a_u64Dst) = (int32_t)u32Tmp; \
6119	} while (0)
6120
6121	#define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
6122	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
6123	#define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
6124	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
6125	#define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
6126	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
6127	#define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
6128	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
6129
6130	#define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
6131	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
6132	#define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
6133	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
6134	#define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
6135	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
6136	#define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
6137	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
6138
6139	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
6140	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
6141	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
6142	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
6143	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u32 = UINT32_C(0xffc00000)
6144	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->au64[0] = UINT64_C(0xfff8000000000000)
6145	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
6146	do { \
6147	(a_pr80Dst)->au64[1] = UINT64_C(0xc000000000000000); \
6148	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
6149	} while (0)
6150
6151
6152	#define IEM_MC_PUSH_U16(a_u16Value) \
6153	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pIemCpu, (a_u16Value)))
6154	#define IEM_MC_PUSH_U32(a_u32Value) \
6155	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pIemCpu, (a_u32Value)))
6156	#define IEM_MC_PUSH_U64(a_u64Value) \
6157	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pIemCpu, (a_u64Value)))
6158
6159	#define IEM_MC_POP_U16(a_pu16Value) \
6160	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pIemCpu, (a_pu16Value)))
6161	#define IEM_MC_POP_U32(a_pu32Value) \
6162	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pIemCpu, (a_pu32Value)))
6163	#define IEM_MC_POP_U64(a_pu64Value) \
6164	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pIemCpu, (a_pu64Value)))
6165
6166	/** Maps guest memory for direct or bounce buffered access.
6167	* The purpose is to pass it to an operand implementation, thus the a_iArg.
6168	* @remarks May return.
6169	*/
6170	#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
6171	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pIemCpu, (void *)&(a_pMem), sizeof((a_pMem)), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
6172
6173	/** Maps guest memory for direct or bounce buffered access.
6174	* The purpose is to pass it to an operand implementation, thus the a_iArg.
6175	* @remarks May return.
6176	*/
6177	#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_iArg) \
6178	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pIemCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
6179
6180	/** Commits the memory and unmaps the guest memory.
6181	* @remarks May return.
6182	*/
6183	#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
6184	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pIemCpu, (a_pvMem), (a_fAccess)))
6185
6186	/** Commits the memory and unmaps the guest memory unless the FPU status word
6187	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
6188	* that would cause FLD not to store.
6189	*
6190	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
6191	* store, while \#P will not.
6192	*
6193	* @remarks May in theory return - for now.
6194	*/
6195	#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
6196	do { \
6197	if ( !(a_u16FSW & X86_FSW_ES) \
6198	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
6199	& ~(pIemCpu->CTX_SUFF(pCtx)->fpu.FCW & X86_FCW_MASK_ALL) ) ) \
6200	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pIemCpu, (a_pvMem), (a_fAccess))); \
6201	} while (0)
6202
6203	/** Calculate efficient address from R/M. */
6204	#define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm) \
6205	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pIemCpu, (bRm), &(a_GCPtrEff)))
6206
6207	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
6208	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
6209	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
6210	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
6211	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
6212
6213	/**
6214	* Defers the rest of the instruction emulation to a C implementation routine
6215	* and returns, only taking the standard parameters.
6216	*
6217	* @param a_pfnCImpl The pointer to the C routine.
6218	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
6219	*/
6220	#define IEM_MC_CALL_CIMPL_0(a_pfnCImpl) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode)
6221
6222	/**
6223	* Defers the rest of instruction emulation to a C implementation routine and
6224	* returns, taking one argument in addition to the standard ones.
6225	*
6226	* @param a_pfnCImpl The pointer to the C routine.
6227	* @param a0 The argument.
6228	*/
6229	#define IEM_MC_CALL_CIMPL_1(a_pfnCImpl, a0) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0)
6230
6231	/**
6232	* Defers the rest of the instruction emulation to a C implementation routine
6233	* and returns, taking two arguments in addition to the standard ones.
6234	*
6235	* @param a_pfnCImpl The pointer to the C routine.
6236	* @param a0 The first extra argument.
6237	* @param a1 The second extra argument.
6238	*/
6239	#define IEM_MC_CALL_CIMPL_2(a_pfnCImpl, a0, a1) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1)
6240
6241	/**
6242	* Defers the rest of the instruction emulation to a C implementation routine
6243	* and returns, taking two arguments in addition to the standard ones.
6244	*
6245	* @param a_pfnCImpl The pointer to the C routine.
6246	* @param a0 The first extra argument.
6247	* @param a1 The second extra argument.
6248	* @param a2 The third extra argument.
6249	*/
6250	#define IEM_MC_CALL_CIMPL_3(a_pfnCImpl, a0, a1, a2) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2)
6251
6252	/**
6253	* Defers the rest of the instruction emulation to a C implementation routine
6254	* and returns, taking two arguments in addition to the standard ones.
6255	*
6256	* @param a_pfnCImpl The pointer to the C routine.
6257	* @param a0 The first extra argument.
6258	* @param a1 The second extra argument.
6259	* @param a2 The third extra argument.
6260	* @param a3 The fourth extra argument.
6261	* @param a4 The fifth extra argument.
6262	*/
6263	#define IEM_MC_CALL_CIMPL_5(a_pfnCImpl, a0, a1, a2, a3, a4) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2, a3, a4)
6264
6265	/**
6266	* Defers the entire instruction emulation to a C implementation routine and
6267	* returns, only taking the standard parameters.
6268	*
6269	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
6270	*
6271	* @param a_pfnCImpl The pointer to the C routine.
6272	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
6273	*/
6274	#define IEM_MC_DEFER_TO_CIMPL_0(a_pfnCImpl) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode)
6275
6276	/**
6277	* Defers the entire instruction emulation to a C implementation routine and
6278	* returns, taking one argument in addition to the standard ones.
6279	*
6280	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
6281	*
6282	* @param a_pfnCImpl The pointer to the C routine.
6283	* @param a0 The argument.
6284	*/
6285	#define IEM_MC_DEFER_TO_CIMPL_1(a_pfnCImpl, a0) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0)
6286
6287	/**
6288	* Defers the entire instruction emulation to a C implementation routine and
6289	* returns, taking two arguments in addition to the standard ones.
6290	*
6291	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
6292	*
6293	* @param a_pfnCImpl The pointer to the C routine.
6294	* @param a0 The first extra argument.
6295	* @param a1 The second extra argument.
6296	*/
6297	#define IEM_MC_DEFER_TO_CIMPL_2(a_pfnCImpl, a0, a1) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1)
6298
6299	/**
6300	* Defers the entire instruction emulation to a C implementation routine and
6301	* returns, taking three arguments in addition to the standard ones.
6302	*
6303	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
6304	*
6305	* @param a_pfnCImpl The pointer to the C routine.
6306	* @param a0 The first extra argument.
6307	* @param a1 The second extra argument.
6308	* @param a2 The third extra argument.
6309	*/
6310	#define IEM_MC_DEFER_TO_CIMPL_3(a_pfnCImpl, a0, a1, a2) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2)
6311
6312	/**
6313	* Calls a FPU assembly implementation taking one visible argument.
6314	*
6315	* @param a_pfnAImpl Pointer to the assembly FPU routine.
6316	* @param a0 The first extra argument.
6317	*/
6318	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
6319	do { \
6320	iemFpuPrepareUsage(pIemCpu); \
6321	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0)); \
6322	} while (0)
6323
6324	/**
6325	* Calls a FPU assembly implementation taking two visible arguments.
6326	*
6327	* @param a_pfnAImpl Pointer to the assembly FPU routine.
6328	* @param a0 The first extra argument.
6329	* @param a1 The second extra argument.
6330	*/
6331	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
6332	do { \
6333	iemFpuPrepareUsage(pIemCpu); \
6334	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1)); \
6335	} while (0)
6336
6337	/**
6338	* Calls a FPU assembly implementation taking three visible arguments.
6339	*
6340	* @param a_pfnAImpl Pointer to the assembly FPU routine.
6341	* @param a0 The first extra argument.
6342	* @param a1 The second extra argument.
6343	* @param a2 The third extra argument.
6344	*/
6345	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
6346	do { \
6347	iemFpuPrepareUsage(pIemCpu); \
6348	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1), (a2)); \
6349	} while (0)
6350
6351	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
6352	do { \
6353	(a_FpuData).FSW = (a_FSW); \
6354	(a_FpuData).r80Result = *(a_pr80Value); \
6355	} while (0)
6356
6357	/** Pushes FPU result onto the stack. */
6358	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData) \
6359	iemFpuPushResult(pIemCpu, &a_FpuData)
6360	/** Pushes FPU result onto the stack and sets the FPUDP. */
6361	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff) \
6362	iemFpuPushResultWithMemOp(pIemCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff)
6363
6364	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
6365	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo) \
6366	iemFpuPushResultTwo(pIemCpu, &a_FpuDataTwo)
6367
6368	/** Stores FPU result in a stack register. */
6369	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg) \
6370	iemFpuStoreResult(pIemCpu, &a_FpuData, a_iStReg)
6371	/** Stores FPU result in a stack register and pops the stack. */
6372	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg) \
6373	iemFpuStoreResultThenPop(pIemCpu, &a_FpuData, a_iStReg)
6374	/** Stores FPU result in a stack register and sets the FPUDP. */
6375	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
6376	iemFpuStoreResultWithMemOp(pIemCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
6377	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
6378	* stack. */
6379	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
6380	iemFpuStoreResultWithMemOpThenPop(pIemCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
6381
6382	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
6383	#define IEM_MC_UPDATE_FPU_OPCODE_IP() \
6384	iemFpuUpdateOpcodeAndIp(pIemCpu)
6385
6386	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
6387	#define IEM_MC_UPDATE_FSW(a_u16FSW) \
6388	iemFpuUpdateFSW(pIemCpu, a_u16FSW)
6389	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
6390	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW) \
6391	iemFpuUpdateFSW(pIemCpu, a_u16FSW)
6392	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
6393	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
6394	iemFpuUpdateFSWWithMemOp(pIemCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
6395	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
6396	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW) \
6397	iemFpuUpdateFSWThenPop(pIemCpu, a_u16FSW)
6398	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
6399	* stack. */
6400	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
6401	iemFpuUpdateFSWWithMemOpThenPop(pIemCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
6402	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
6403	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW) \
6404	iemFpuUpdateFSWThenPop(pIemCpu, a_u16FSW)
6405
6406	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
6407	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst) \
6408	iemFpuStackUnderflow(pIemCpu, a_iStDst)
6409	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
6410	* stack. */
6411	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst) \
6412	iemFpuStackUnderflowThenPop(pIemCpu, a_iStDst)
6413	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
6414	* FPUDS. */
6415	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
6416	iemFpuStackUnderflowWithMemOp(pIemCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
6417	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
6418	* FPUDS. Pops stack. */
6419	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
6420	iemFpuStackUnderflowWithMemOpThenPop(pIemCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
6421	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
6422	* stack twice. */
6423	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP() \
6424	iemFpuStackUnderflowThenPopPop(pIemCpu)
6425	/** Raises a FPU stack underflow exception for an instruction pushing a result
6426	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
6427	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW() \
6428	iemFpuStackPushUnderflow(pIemCpu)
6429	/** Raises a FPU stack underflow exception for an instruction pushing a result
6430	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
6431	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO() \
6432	iemFpuStackPushUnderflowTwo(pIemCpu)
6433
6434	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
6435	* FPUIP, FPUCS and FOP. */
6436	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW() \
6437	iemFpuStackPushOverflow(pIemCpu)
6438	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
6439	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
6440	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff) \
6441	iemFpuStackPushOverflowWithMemOp(pIemCpu, a_iEffSeg, a_GCPtrEff)
6442
6443
6444	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) {
6445	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit))) {
6446	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBits)) {
6447	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBits))) {
6448	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
6449	if ( !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6450	!= !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6451	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
6452	if ( !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6453	== !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6454	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
6455	if ( (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) \
6456	\|\| !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6457	!= !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6458	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
6459	if ( !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) \
6460	&& !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6461	== !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6462	#define IEM_MC_IF_CX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->cx != 0) {
6463	#define IEM_MC_IF_ECX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->ecx != 0) {
6464	#define IEM_MC_IF_RCX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->rcx != 0) {
6465	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
6466	if ( pIemCpu->CTX_SUFF(pCtx)->cx != 0 \
6467	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6468	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
6469	if ( pIemCpu->CTX_SUFF(pCtx)->ecx != 0 \
6470	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6471	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
6472	if ( pIemCpu->CTX_SUFF(pCtx)->rcx != 0 \
6473	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6474	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
6475	if ( pIemCpu->CTX_SUFF(pCtx)->cx != 0 \
6476	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6477	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
6478	if ( pIemCpu->CTX_SUFF(pCtx)->ecx != 0 \
6479	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6480	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
6481	if ( pIemCpu->CTX_SUFF(pCtx)->rcx != 0 \
6482	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6483	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
6484	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if ((uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
6485	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
6486	if (iemFpuStRegNotEmpty(pIemCpu, (a_iSt)) == VINF_SUCCESS) {
6487	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
6488	if (iemFpuStRegNotEmpty(pIemCpu, (a_iSt)) != VINF_SUCCESS) {
6489	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
6490	if (iemFpuStRegNotEmptyRef(pIemCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
6491	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
6492	if (iemFpu2StRegsNotEmptyRef(pIemCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
6493	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
6494	if (iemFpu2StRegsNotEmptyRefFirst(pIemCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
6495	#define IEM_MC_IF_FCW_IM() \
6496	if (pIemCpu->CTX_SUFF(pCtx)->fpu.FCW & X86_FCW_IM) {
6497
6498	#define IEM_MC_ELSE() } else {
6499	#define IEM_MC_ENDIF() } do {} while (0)
6500
6501	/** @} */
6502
6503
6504	/** @name Opcode Debug Helpers.
6505	* @{
6506	*/
6507	#ifdef DEBUG
6508	# define IEMOP_MNEMONIC(a_szMnemonic) \
6509	Log4(("decode - %04x:%RGv %s%s [#%u]\n", pIemCpu->CTX_SUFF(pCtx)->cs, pIemCpu->CTX_SUFF(pCtx)->rip, \
6510	pIemCpu->fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, pIemCpu->cInstructions))
6511	# define IEMOP_MNEMONIC2(a_szMnemonic, a_szOps) \
6512	Log4(("decode - %04x:%RGv %s%s %s [#%u]\n", pIemCpu->CTX_SUFF(pCtx)->cs, pIemCpu->CTX_SUFF(pCtx)->rip, \
6513	pIemCpu->fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, a_szOps, pIemCpu->cInstructions))
6514	#else
6515	# define IEMOP_MNEMONIC(a_szMnemonic) do { } while (0)
6516	# define IEMOP_MNEMONIC2(a_szMnemonic, a_szOps) do { } while (0)
6517	#endif
6518
6519	/** @} */
6520
6521
6522	/** @name Opcode Helpers.
6523	* @{
6524	*/
6525
6526	/** The instruction allows no lock prefixing (in this encoding), throw #UD if
6527	* lock prefixed.
6528	* @deprecated IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX */
6529	#define IEMOP_HLP_NO_LOCK_PREFIX() \
6530	do \
6531	{ \
6532	if (pIemCpu->fPrefixes & IEM_OP_PRF_LOCK) \
6533	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
6534	} while (0)
6535
6536	/** The instruction is not available in 64-bit mode, throw #UD if we're in
6537	* 64-bit mode. */
6538	#define IEMOP_HLP_NO_64BIT() \
6539	do \
6540	{ \
6541	if (pIemCpu->enmCpuMode == IEMMODE_64BIT) \
6542	return IEMOP_RAISE_INVALID_OPCODE(); \
6543	} while (0)
6544
6545	/** The instruction defaults to 64-bit operand size if 64-bit mode. */
6546	#define IEMOP_HLP_DEFAULT_64BIT_OP_SIZE() \
6547	do \
6548	{ \
6549	if (pIemCpu->enmCpuMode == IEMMODE_64BIT) \
6550	iemRecalEffOpSize64Default(pIemCpu); \
6551	} while (0)
6552
6553	/**
6554	* Done decoding.
6555	*/
6556	#define IEMOP_HLP_DONE_DECODING() \
6557	do \
6558	{ \
6559	/nothing for now, maybe later... / \
6560	} while (0)
6561
6562	/**
6563	* Done decoding, raise \#UD exception if lock prefix present.
6564	*/
6565	#define IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX() \
6566	do \
6567	{ \
6568	if (pIemCpu->fPrefixes & IEM_OP_PRF_LOCK) \
6569	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
6570	} while (0)
6571
6572
6573	/**
6574	* Calculates the effective address of a ModR/M memory operand.
6575	*
6576	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
6577	*
6578	* @return Strict VBox status code.
6579	* @param pIemCpu The IEM per CPU data.
6580	* @param bRm The ModRM byte.
6581	* @param pGCPtrEff Where to return the effective address.
6582	*/
6583	static VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PIEMCPU pIemCpu, uint8_t bRm, PRTGCPTR pGCPtrEff)
6584	{
6585	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
6586	PCCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
6587	#define SET_SS_DEF() \
6588	do \
6589	{ \
6590	if (!(pIemCpu->fPrefixes & IEM_OP_PRF_SEG_MASK)) \
6591	pIemCpu->iEffSeg = X86_SREG_SS; \
6592	} while (0)
6593
6594	/** @todo Check the effective address size crap! */
6595	switch (pIemCpu->enmEffAddrMode)
6596	{
6597	case IEMMODE_16BIT:
6598	{
6599	uint16_t u16EffAddr;
6600
6601	/* Handle the disp16 form with no registers first. */
6602	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
6603	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
6604	else
6605	{
6606	/* Get the displacment. */
6607	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
6608	{
6609	case 0: u16EffAddr = 0; break;
6610	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
6611	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
6612	default: AssertFailedReturn(VERR_INTERNAL_ERROR_2); /* (caller checked for these) */
6613	}
6614
6615	/* Add the base and index registers to the disp. */
6616	switch (bRm & X86_MODRM_RM_MASK)
6617	{
6618	case 0: u16EffAddr += pCtx->bx + pCtx->si; break;
6619	case 1: u16EffAddr += pCtx->bx + pCtx->di; break;
6620	case 2: u16EffAddr += pCtx->bp + pCtx->si; SET_SS_DEF(); break;
6621	case 3: u16EffAddr += pCtx->bp + pCtx->di; SET_SS_DEF(); break;
6622	case 4: u16EffAddr += pCtx->si; break;
6623	case 5: u16EffAddr += pCtx->di; break;
6624	case 6: u16EffAddr += pCtx->bp; SET_SS_DEF(); break;
6625	case 7: u16EffAddr += pCtx->bx; break;
6626	}
6627	}
6628
6629	*pGCPtrEff = u16EffAddr;
6630	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#06RGv\n", *pGCPtrEff));
6631	return VINF_SUCCESS;
6632	}
6633
6634	case IEMMODE_32BIT:
6635	{
6636	uint32_t u32EffAddr;
6637
6638	/* Handle the disp32 form with no registers first. */
6639	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
6640	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
6641	else
6642	{
6643	/* Get the register (or SIB) value. */
6644	switch ((bRm & X86_MODRM_RM_MASK))
6645	{
6646	case 0: u32EffAddr = pCtx->eax; break;
6647	case 1: u32EffAddr = pCtx->ecx; break;
6648	case 2: u32EffAddr = pCtx->edx; break;
6649	case 3: u32EffAddr = pCtx->ebx; break;
6650	case 4: /* SIB */
6651	{
6652	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
6653
6654	/* Get the index and scale it. */
6655	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
6656	{
6657	case 0: u32EffAddr = pCtx->eax; break;
6658	case 1: u32EffAddr = pCtx->ecx; break;
6659	case 2: u32EffAddr = pCtx->edx; break;
6660	case 3: u32EffAddr = pCtx->ebx; break;
6661	case 4: u32EffAddr = 0; /none / break;
6662	case 5: u32EffAddr = pCtx->ebp; break;
6663	case 6: u32EffAddr = pCtx->esi; break;
6664	case 7: u32EffAddr = pCtx->edi; break;
6665	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6666	}
6667	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
6668
6669	/* add base */
6670	switch (bSib & X86_SIB_BASE_MASK)
6671	{
6672	case 0: u32EffAddr += pCtx->eax; break;
6673	case 1: u32EffAddr += pCtx->ecx; break;
6674	case 2: u32EffAddr += pCtx->edx; break;
6675	case 3: u32EffAddr += pCtx->ebx; break;
6676	case 4: u32EffAddr += pCtx->esp; SET_SS_DEF(); break;
6677	case 5:
6678	if ((bRm & X86_MODRM_MOD_MASK) != 0)
6679	{
6680	u32EffAddr += pCtx->ebp;
6681	SET_SS_DEF();
6682	}
6683	else
6684	{
6685	uint32_t u32Disp;
6686	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6687	u32EffAddr += u32Disp;
6688	}
6689	break;
6690	case 6: u32EffAddr += pCtx->esi; break;
6691	case 7: u32EffAddr += pCtx->edi; break;
6692	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6693	}
6694	break;
6695	}
6696	case 5: u32EffAddr = pCtx->ebp; SET_SS_DEF(); break;
6697	case 6: u32EffAddr = pCtx->esi; break;
6698	case 7: u32EffAddr = pCtx->edi; break;
6699	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6700	}
6701
6702	/* Get and add the displacement. */
6703	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
6704	{
6705	case 0:
6706	break;
6707	case 1:
6708	{
6709	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
6710	u32EffAddr += i8Disp;
6711	break;
6712	}
6713	case 2:
6714	{
6715	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6716	u32EffAddr += u32Disp;
6717	break;
6718	}
6719	default:
6720	AssertFailedReturn(VERR_INTERNAL_ERROR_2); /* (caller checked for these) */
6721	}
6722
6723	}
6724	if (pIemCpu->enmEffAddrMode == IEMMODE_32BIT)
6725	*pGCPtrEff = u32EffAddr;
6726	else
6727	{
6728	Assert(pIemCpu->enmEffAddrMode == IEMMODE_16BIT);
6729	*pGCPtrEff = u32EffAddr & UINT16_MAX;
6730	}
6731	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
6732	return VINF_SUCCESS;
6733	}
6734
6735	case IEMMODE_64BIT:
6736	{
6737	uint64_t u64EffAddr;
6738
6739	/* Handle the rip+disp32 form with no registers first. */
6740	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
6741	{
6742	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
6743	u64EffAddr += pCtx->rip + pIemCpu->offOpcode;
6744	}
6745	else
6746	{
6747	/* Get the register (or SIB) value. */
6748	switch ((bRm & X86_MODRM_RM_MASK) \| pIemCpu->uRexB)
6749	{
6750	case 0: u64EffAddr = pCtx->rax; break;
6751	case 1: u64EffAddr = pCtx->rcx; break;
6752	case 2: u64EffAddr = pCtx->rdx; break;
6753	case 3: u64EffAddr = pCtx->rbx; break;
6754	case 5: u64EffAddr = pCtx->rbp; SET_SS_DEF(); break;
6755	case 6: u64EffAddr = pCtx->rsi; break;
6756	case 7: u64EffAddr = pCtx->rdi; break;
6757	case 8: u64EffAddr = pCtx->r8; break;
6758	case 9: u64EffAddr = pCtx->r9; break;
6759	case 10: u64EffAddr = pCtx->r10; break;
6760	case 11: u64EffAddr = pCtx->r11; break;
6761	case 13: u64EffAddr = pCtx->r13; break;
6762	case 14: u64EffAddr = pCtx->r14; break;
6763	case 15: u64EffAddr = pCtx->r15; break;
6764	/* SIB */
6765	case 4:
6766	case 12:
6767	{
6768	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
6769
6770	/* Get the index and scale it. */
6771	switch (((bSib & X86_SIB_INDEX_SHIFT) >> X86_SIB_INDEX_SMASK) \| pIemCpu->uRexIndex)
6772	{
6773	case 0: u64EffAddr = pCtx->rax; break;
6774	case 1: u64EffAddr = pCtx->rcx; break;
6775	case 2: u64EffAddr = pCtx->rdx; break;
6776	case 3: u64EffAddr = pCtx->rbx; break;
6777	case 4: u64EffAddr = 0; /none / break;
6778	case 5: u64EffAddr = pCtx->rbp; break;
6779	case 6: u64EffAddr = pCtx->rsi; break;
6780	case 7: u64EffAddr = pCtx->rdi; break;
6781	case 8: u64EffAddr = pCtx->r8; break;
6782	case 9: u64EffAddr = pCtx->r9; break;
6783	case 10: u64EffAddr = pCtx->r10; break;
6784	case 11: u64EffAddr = pCtx->r11; break;
6785	case 12: u64EffAddr = pCtx->r12; break;
6786	case 13: u64EffAddr = pCtx->r13; break;
6787	case 14: u64EffAddr = pCtx->r14; break;
6788	case 15: u64EffAddr = pCtx->r15; break;
6789	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6790	}
6791	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
6792
6793	/* add base */
6794	switch ((bSib & X86_SIB_BASE_MASK) \| pIemCpu->uRexB)
6795	{
6796	case 0: u64EffAddr += pCtx->rax; break;
6797	case 1: u64EffAddr += pCtx->rcx; break;
6798	case 2: u64EffAddr += pCtx->rdx; break;
6799	case 3: u64EffAddr += pCtx->rbx; break;
6800	case 4: u64EffAddr += pCtx->rsp; SET_SS_DEF(); break;
6801	case 6: u64EffAddr += pCtx->rsi; break;
6802	case 7: u64EffAddr += pCtx->rdi; break;
6803	case 8: u64EffAddr += pCtx->r8; break;
6804	case 9: u64EffAddr += pCtx->r9; break;
6805	case 10: u64EffAddr += pCtx->r10; break;
6806	case 11: u64EffAddr += pCtx->r11; break;
6807	case 14: u64EffAddr += pCtx->r14; break;
6808	case 15: u64EffAddr += pCtx->r15; break;
6809	/* complicated encodings */
6810	case 5:
6811	case 13:
6812	if ((bRm & X86_MODRM_MOD_MASK) != 0)
6813	{
6814	if (!pIemCpu->uRexB)
6815	{
6816	u64EffAddr += pCtx->rbp;
6817	SET_SS_DEF();
6818	}
6819	else
6820	u64EffAddr += pCtx->r13;
6821	}
6822	else
6823	{
6824	uint32_t u32Disp;
6825	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6826	u64EffAddr += (int32_t)u32Disp;
6827	}
6828	break;
6829	}
6830	break;
6831	}
6832	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6833	}
6834
6835	/* Get and add the displacement. */
6836	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
6837	{
6838	case 0:
6839	break;
6840	case 1:
6841	{
6842	int8_t i8Disp;
6843	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
6844	u64EffAddr += i8Disp;
6845	break;
6846	}
6847	case 2:
6848	{
6849	uint32_t u32Disp;
6850	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6851	u64EffAddr += (int32_t)u32Disp;
6852	break;
6853	}
6854	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
6855	}
6856
6857	}
6858	if (pIemCpu->enmEffAddrMode == IEMMODE_64BIT)
6859	*pGCPtrEff = u64EffAddr;
6860	else
6861	*pGCPtrEff = u64EffAddr & UINT16_MAX;
6862	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
6863	return VINF_SUCCESS;
6864	}
6865	}
6866
6867	AssertFailedReturn(VERR_INTERNAL_ERROR_3);
6868	}
6869
6870	/** @} */
6871
6872
6873
6874	/*
6875	* Include the instructions
6876	*/
6877	#include "IEMAllInstructions.cpp.h"
6878
6879
6880
6881
6882	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
6883
6884	/**
6885	* Sets up execution verification mode.
6886	*/
6887	static void iemExecVerificationModeSetup(PIEMCPU pIemCpu)
6888	{
6889	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
6890	PCPUMCTX pOrgCtx = pIemCpu->CTX_SUFF(pCtx);
6891
6892	/*
6893	* Enable verification and/or logging.
6894	*/
6895	pIemCpu->fNoRem = !LogIs6Enabled(); /* logging triggers the no-rem/rem verification stuff */
6896	if ( pIemCpu->fNoRem
6897	#if 0 /* auto enable on first paged protected mode interrupt */
6898	&& pOrgCtx->eflags.Bits.u1IF
6899	&& (pOrgCtx->cr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG)
6900	&& TRPMHasTrap(pVCpu)
6901	&& EMGetInhibitInterruptsPC(pVCpu) != pOrgCtx->rip)
6902	#endif
6903	#if 0
6904	&& pOrgCtx->cs == 0x10
6905	&& ( pOrgCtx->rip == 0x90119e3e
6906	\|\| pOrgCtx->rip == 0x901d9810
6907	)
6908	#endif
6909	#if 0 /* Auto enable DSL - FPU stuff. */
6910	&& pOrgCtx->cs == 0x10
6911	&& (// pOrgCtx->rip == 0xc02ec07f
6912	//\|\| pOrgCtx->rip == 0xc02ec082
6913	//\|\| pOrgCtx->rip == 0xc02ec0c9
6914	0
6915	\|\| pOrgCtx->rip == 0x0c010e7c4 /* fxsave */
6916	)
6917	#endif
6918	#if 1 /* Auto enable DSL - fstp st0 stuff. */
6919	&& pOrgCtx->cs == 0x23
6920	&& pOrgCtx->rip == 0x804aff7
6921	#endif
6922	#if 0
6923	&& pOrgCtx->rip == 0x9022bb3a
6924	#endif
6925	#if 0
6926	&& 0
6927	#endif
6928	)
6929	{
6930	RTLogGroupSettings(NULL, "iem.eo.l6.l2");
6931	RTLogFlags(NULL, "enabled");
6932	pIemCpu->fNoRem = false;
6933	}
6934
6935	/*
6936	* Switch state.
6937	*/
6938	if (IEM_VERIFICATION_ENABLED(pIemCpu))
6939	{
6940	static CPUMCTX s_DebugCtx; /* Ugly! */
6941
6942	s_DebugCtx = *pOrgCtx;
6943	pIemCpu->CTX_SUFF(pCtx) = &s_DebugCtx;
6944	}
6945
6946	/*
6947	* See if there is an interrupt pending in TRPM and inject it if we can.
6948	*/
6949	if ( pOrgCtx->eflags.Bits.u1IF
6950	&& TRPMHasTrap(pVCpu)
6951	&& EMGetInhibitInterruptsPC(pVCpu) != pOrgCtx->rip)
6952	{
6953	uint8_t u8TrapNo;
6954	TRPMEVENT enmType;
6955	RTGCUINT uErrCode;
6956	RTGCPTR uCr2;
6957	int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2); AssertRC(rc2);
6958	IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2);
6959	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
6960	TRPMResetTrap(pVCpu);
6961	}
6962
6963	/*
6964	* Reset the counters.
6965	*/
6966	pIemCpu->cIOReads = 0;
6967	pIemCpu->cIOWrites = 0;
6968	pIemCpu->fUndefinedEFlags = 0;
6969
6970	if (IEM_VERIFICATION_ENABLED(pIemCpu))
6971	{
6972	/*
6973	* Free all verification records.
6974	*/
6975	PIEMVERIFYEVTREC pEvtRec = pIemCpu->pIemEvtRecHead;
6976	pIemCpu->pIemEvtRecHead = NULL;
6977	pIemCpu->ppIemEvtRecNext = &pIemCpu->pIemEvtRecHead;
6978	do
6979	{
6980	while (pEvtRec)
6981	{
6982	PIEMVERIFYEVTREC pNext = pEvtRec->pNext;
6983	pEvtRec->pNext = pIemCpu->pFreeEvtRec;
6984	pIemCpu->pFreeEvtRec = pEvtRec;
6985	pEvtRec = pNext;
6986	}
6987	pEvtRec = pIemCpu->pOtherEvtRecHead;
6988	pIemCpu->pOtherEvtRecHead = NULL;
6989	pIemCpu->ppOtherEvtRecNext = &pIemCpu->pOtherEvtRecHead;
6990	} while (pEvtRec);
6991	}
6992	}
6993
6994
6995	/**
6996	* Allocate an event record.
6997	* @returns Poitner to a record.
6998	*/
6999	static PIEMVERIFYEVTREC iemVerifyAllocRecord(PIEMCPU pIemCpu)
7000	{
7001	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
7002	return NULL;
7003
7004	PIEMVERIFYEVTREC pEvtRec = pIemCpu->pFreeEvtRec;
7005	if (pEvtRec)
7006	pIemCpu->pFreeEvtRec = pEvtRec->pNext;
7007	else
7008	{
7009	if (!pIemCpu->ppIemEvtRecNext)
7010	return NULL; /* Too early (fake PCIBIOS), ignore notification. */
7011
7012	pEvtRec = (PIEMVERIFYEVTREC)MMR3HeapAlloc(IEMCPU_TO_VM(pIemCpu), MM_TAG_EM /* lazy bird/, sizeof(pEvtRec));
7013	if (!pEvtRec)
7014	return NULL;
7015	}
7016	pEvtRec->enmEvent = IEMVERIFYEVENT_INVALID;
7017	pEvtRec->pNext = NULL;
7018	return pEvtRec;
7019	}
7020
7021
7022	/**
7023	* IOMMMIORead notification.
7024	*/
7025	VMM_INT_DECL(void) IEMNotifyMMIORead(PVM pVM, RTGCPHYS GCPhys, size_t cbValue)
7026	{
7027	PVMCPU pVCpu = VMMGetCpu(pVM);
7028	if (!pVCpu)
7029	return;
7030	PIEMCPU pIemCpu = &pVCpu->iem.s;
7031	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7032	if (!pEvtRec)
7033	return;
7034	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
7035	pEvtRec->u.RamRead.GCPhys = GCPhys;
7036	pEvtRec->u.RamRead.cb = (uint32_t)cbValue;
7037	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
7038	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
7039	}
7040
7041
7042	/**
7043	* IOMMMIOWrite notification.
7044	*/
7045	VMM_INT_DECL(void) IEMNotifyMMIOWrite(PVM pVM, RTGCPHYS GCPhys, uint32_t u32Value, size_t cbValue)
7046	{
7047	PVMCPU pVCpu = VMMGetCpu(pVM);
7048	if (!pVCpu)
7049	return;
7050	PIEMCPU pIemCpu = &pVCpu->iem.s;
7051	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7052	if (!pEvtRec)
7053	return;
7054	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
7055	pEvtRec->u.RamWrite.GCPhys = GCPhys;
7056	pEvtRec->u.RamWrite.cb = (uint32_t)cbValue;
7057	pEvtRec->u.RamWrite.ab[0] = RT_BYTE1(u32Value);
7058	pEvtRec->u.RamWrite.ab[1] = RT_BYTE2(u32Value);
7059	pEvtRec->u.RamWrite.ab[2] = RT_BYTE3(u32Value);
7060	pEvtRec->u.RamWrite.ab[3] = RT_BYTE4(u32Value);
7061	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
7062	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
7063	}
7064
7065
7066	/**
7067	* IOMIOPortRead notification.
7068	*/
7069	VMM_INT_DECL(void) IEMNotifyIOPortRead(PVM pVM, RTIOPORT Port, size_t cbValue)
7070	{
7071	PVMCPU pVCpu = VMMGetCpu(pVM);
7072	if (!pVCpu)
7073	return;
7074	PIEMCPU pIemCpu = &pVCpu->iem.s;
7075	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7076	if (!pEvtRec)
7077	return;
7078	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_READ;
7079	pEvtRec->u.IOPortRead.Port = Port;
7080	pEvtRec->u.IOPortRead.cbValue = (uint32_t)cbValue;
7081	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
7082	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
7083	}
7084
7085	/**
7086	* IOMIOPortWrite notification.
7087	*/
7088	VMM_INT_DECL(void) IEMNotifyIOPortWrite(PVM pVM, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
7089	{
7090	PVMCPU pVCpu = VMMGetCpu(pVM);
7091	if (!pVCpu)
7092	return;
7093	PIEMCPU pIemCpu = &pVCpu->iem.s;
7094	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7095	if (!pEvtRec)
7096	return;
7097	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_WRITE;
7098	pEvtRec->u.IOPortWrite.Port = Port;
7099	pEvtRec->u.IOPortWrite.cbValue = (uint32_t)cbValue;
7100	pEvtRec->u.IOPortWrite.u32Value = u32Value;
7101	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
7102	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
7103	}
7104
7105
7106	VMM_INT_DECL(void) IEMNotifyIOPortReadString(PVM pVM, RTIOPORT Port, RTGCPTR GCPtrDst, RTGCUINTREG cTransfers, size_t cbValue)
7107	{
7108	AssertFailed();
7109	}
7110
7111
7112	VMM_INT_DECL(void) IEMNotifyIOPortWriteString(PVM pVM, RTIOPORT Port, RTGCPTR GCPtrSrc, RTGCUINTREG cTransfers, size_t cbValue)
7113	{
7114	AssertFailed();
7115	}
7116
7117
7118	/**
7119	* Fakes and records an I/O port read.
7120	*
7121	* @returns VINF_SUCCESS.
7122	* @param pIemCpu The IEM per CPU data.
7123	* @param Port The I/O port.
7124	* @param pu32Value Where to store the fake value.
7125	* @param cbValue The size of the access.
7126	*/
7127	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue)
7128	{
7129	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7130	if (pEvtRec)
7131	{
7132	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_READ;
7133	pEvtRec->u.IOPortRead.Port = Port;
7134	pEvtRec->u.IOPortRead.cbValue = (uint32_t)cbValue;
7135	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
7136	*pIemCpu->ppIemEvtRecNext = pEvtRec;
7137	}
7138	pIemCpu->cIOReads++;
7139	*pu32Value = 0xcccccccc;
7140	return VINF_SUCCESS;
7141	}
7142
7143
7144	/**
7145	* Fakes and records an I/O port write.
7146	*
7147	* @returns VINF_SUCCESS.
7148	* @param pIemCpu The IEM per CPU data.
7149	* @param Port The I/O port.
7150	* @param u32Value The value being written.
7151	* @param cbValue The size of the access.
7152	*/
7153	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
7154	{
7155	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7156	if (pEvtRec)
7157	{
7158	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_WRITE;
7159	pEvtRec->u.IOPortWrite.Port = Port;
7160	pEvtRec->u.IOPortWrite.cbValue = (uint32_t)cbValue;
7161	pEvtRec->u.IOPortWrite.u32Value = u32Value;
7162	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
7163	*pIemCpu->ppIemEvtRecNext = pEvtRec;
7164	}
7165	pIemCpu->cIOWrites++;
7166	return VINF_SUCCESS;
7167	}
7168
7169
7170	/**
7171	* Used to add extra details about a stub case.
7172	* @param pIemCpu The IEM per CPU state.
7173	*/
7174	static void iemVerifyAssertMsg2(PIEMCPU pIemCpu)
7175	{
7176	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7177	PVM pVM = IEMCPU_TO_VM(pIemCpu);
7178	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
7179	char szRegs[4096];
7180	DBGFR3RegPrintf(pVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
7181	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
7182	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
7183	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
7184	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
7185	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
7186	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
7187	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
7188	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
7189	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
7190	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
7191	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
7192	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
7193	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
7194	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
7195	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
7196	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
7197	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
7198	" efer=%016VR{efer}\n"
7199	" pat=%016VR{pat}\n"
7200	" sf_mask=%016VR{sf_mask}\n"
7201	"krnl_gs_base=%016VR{krnl_gs_base}\n"
7202	" lstar=%016VR{lstar}\n"
7203	" star=%016VR{star} cstar=%016VR{cstar}\n"
7204	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
7205	);
7206
7207	char szInstr1[256];
7208	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip - pIemCpu->offOpcode,
7209	DBGF_DISAS_FLAGS_DEFAULT_MODE,
7210	szInstr1, sizeof(szInstr1), NULL);
7211	char szInstr2[256];
7212	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0,
7213	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
7214	szInstr2, sizeof(szInstr2), NULL);
7215
7216	RTAssertMsg2Weak("%s%s\n%s\n", szRegs, szInstr1, szInstr2);
7217	}
7218
7219
7220	/**
7221	* Used by iemVerifyAssertRecord and iemVerifyAssertRecords to add a record
7222	* dump to the assertion info.
7223	*
7224	* @param pEvtRec The record to dump.
7225	*/
7226	static void iemVerifyAssertAddRecordDump(PIEMVERIFYEVTREC pEvtRec)
7227	{
7228	switch (pEvtRec->enmEvent)
7229	{
7230	case IEMVERIFYEVENT_IOPORT_READ:
7231	RTAssertMsg2Add("I/O PORT READ from %#6x, %d bytes\n",
7232	pEvtRec->u.IOPortWrite.Port,
7233	pEvtRec->u.IOPortWrite.cbValue);
7234	break;
7235	case IEMVERIFYEVENT_IOPORT_WRITE:
7236	RTAssertMsg2Add("I/O PORT WRITE to %#6x, %d bytes, value %#x\n",
7237	pEvtRec->u.IOPortWrite.Port,
7238	pEvtRec->u.IOPortWrite.cbValue,
7239	pEvtRec->u.IOPortWrite.u32Value);
7240	break;
7241	case IEMVERIFYEVENT_RAM_READ:
7242	RTAssertMsg2Add("RAM READ at %RGp, %#4zx bytes\n",
7243	pEvtRec->u.RamRead.GCPhys,
7244	pEvtRec->u.RamRead.cb);
7245	break;
7246	case IEMVERIFYEVENT_RAM_WRITE:
7247	RTAssertMsg2Add("RAM WRITE at %RGp, %#4zx bytes: %.*Rhxs\n",
7248	pEvtRec->u.RamWrite.GCPhys,
7249	pEvtRec->u.RamWrite.cb,
7250	(int)pEvtRec->u.RamWrite.cb,
7251	pEvtRec->u.RamWrite.ab);
7252	break;
7253	default:
7254	AssertMsgFailed(("Invalid event type %d\n", pEvtRec->enmEvent));
7255	break;
7256	}
7257	}
7258
7259
7260	/**
7261	* Raises an assertion on the specified record, showing the given message with
7262	* a record dump attached.
7263	*
7264	* @param pIemCpu The IEM per CPU data.
7265	* @param pEvtRec1 The first record.
7266	* @param pEvtRec2 The second record.
7267	* @param pszMsg The message explaining why we're asserting.
7268	*/
7269	static void iemVerifyAssertRecords(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec1, PIEMVERIFYEVTREC pEvtRec2, const char *pszMsg)
7270	{
7271	RTAssertMsg1(pszMsg, __LINE__, __FILE__, __PRETTY_FUNCTION__);
7272	iemVerifyAssertAddRecordDump(pEvtRec1);
7273	iemVerifyAssertAddRecordDump(pEvtRec2);
7274	iemVerifyAssertMsg2(pIemCpu);
7275	RTAssertPanic();
7276	}
7277
7278
7279	/**
7280	* Raises an assertion on the specified record, showing the given message with
7281	* a record dump attached.
7282	*
7283	* @param pIemCpu The IEM per CPU data.
7284	* @param pEvtRec1 The first record.
7285	* @param pszMsg The message explaining why we're asserting.
7286	*/
7287	static void iemVerifyAssertRecord(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec, const char *pszMsg)
7288	{
7289	RTAssertMsg1(pszMsg, __LINE__, __FILE__, __PRETTY_FUNCTION__);
7290	iemVerifyAssertAddRecordDump(pEvtRec);
7291	iemVerifyAssertMsg2(pIemCpu);
7292	RTAssertPanic();
7293	}
7294
7295
7296	/**
7297	* Verifies a write record.
7298	*
7299	* @param pIemCpu The IEM per CPU data.
7300	* @param pEvtRec The write record.
7301	*/
7302	static void iemVerifyWriteRecord(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec)
7303	{
7304	uint8_t abBuf[sizeof(pEvtRec->u.RamWrite.ab)]; RT_ZERO(abBuf);
7305	Assert(sizeof(abBuf) >= pEvtRec->u.RamWrite.cb);
7306	int rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), abBuf, pEvtRec->u.RamWrite.GCPhys, pEvtRec->u.RamWrite.cb);
7307	if ( RT_FAILURE(rc)
7308	\|\| memcmp(abBuf, pEvtRec->u.RamWrite.ab, pEvtRec->u.RamWrite.cb) )
7309	{
7310	/* fend off ins */
7311	if ( !pIemCpu->cIOReads
7312	\|\| pEvtRec->u.RamWrite.ab[0] != 0xcc
7313	\|\| ( pEvtRec->u.RamWrite.cb != 1
7314	&& pEvtRec->u.RamWrite.cb != 2
7315	&& pEvtRec->u.RamWrite.cb != 4) )
7316	{
7317	/* fend off ROMs */
7318	if ( pEvtRec->u.RamWrite.GCPhys - UINT32_C(0x000c0000) > UINT32_C(0x8000)
7319	&& pEvtRec->u.RamWrite.GCPhys - UINT32_C(0x000e0000) > UINT32_C(0x20000)
7320	&& pEvtRec->u.RamWrite.GCPhys - UINT32_C(0xfffc0000) > UINT32_C(0x40000) )
7321	{
7322	/* fend off fxsave */
7323	if (pEvtRec->u.RamWrite.cb != 512)
7324	{
7325	RTAssertMsg1(NULL, __LINE__, __FILE__, __PRETTY_FUNCTION__);
7326	RTAssertMsg2Weak("Memory at %RGv differs\n", pEvtRec->u.RamWrite.GCPhys);
7327	RTAssertMsg2Add("REM: %.*Rhxs\n"
7328	"IEM: %.*Rhxs\n",
7329	pEvtRec->u.RamWrite.cb, abBuf,
7330	pEvtRec->u.RamWrite.cb, pEvtRec->u.RamWrite.ab);
7331	iemVerifyAssertAddRecordDump(pEvtRec);
7332	iemVerifyAssertMsg2(pIemCpu);
7333	RTAssertPanic();
7334	}
7335	}
7336	}
7337	}
7338
7339	}
7340
7341	/**
7342	* Performs the post-execution verfication checks.
7343	*/
7344	static void iemExecVerificationModeCheck(PIEMCPU pIemCpu)
7345	{
7346	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
7347	return;
7348
7349	/*
7350	* Switch back the state.
7351	*/
7352	PCPUMCTX pOrgCtx = CPUMQueryGuestCtxPtr(IEMCPU_TO_VMCPU(pIemCpu));
7353	PCPUMCTX pDebugCtx = pIemCpu->CTX_SUFF(pCtx);
7354	Assert(pOrgCtx != pDebugCtx);
7355	pIemCpu->CTX_SUFF(pCtx) = pOrgCtx;
7356
7357	/*
7358	* Execute the instruction in REM.
7359	*/
7360	PVM pVM = IEMCPU_TO_VM(pIemCpu);
7361	EMRemLock(pVM);
7362	int rc = REMR3EmulateInstruction(pVM, IEMCPU_TO_VMCPU(pIemCpu));
7363	AssertRC(rc);
7364	EMRemUnlock(pVM);
7365
7366	/*
7367	* Compare the register states.
7368	*/
7369	unsigned cDiffs = 0;
7370	if (memcmp(pOrgCtx, pDebugCtx, sizeof(*pDebugCtx)))
7371	{
7372	Log(("REM and IEM ends up with different registers!\n"));
7373
7374	# define CHECK_FIELD(a_Field) \
7375	do \
7376	{ \
7377	if (pOrgCtx->a_Field != pDebugCtx->a_Field) \
7378	{ \
7379	switch (sizeof(pOrgCtx->a_Field)) \
7380	{ \
7381	case 1: RTAssertMsg2Weak(" %8s differs - iem=%02x - rem=%02x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7382	case 2: RTAssertMsg2Weak(" %8s differs - iem=%04x - rem=%04x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7383	case 4: RTAssertMsg2Weak(" %8s differs - iem=%08x - rem=%08x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7384	case 8: RTAssertMsg2Weak(" %8s differs - iem=%016llx - rem=%016llx\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7385	default: RTAssertMsg2Weak(" %8s differs\n", #a_Field); break; \
7386	} \
7387	cDiffs++; \
7388	} \
7389	} while (0)
7390
7391	# define CHECK_BIT_FIELD(a_Field) \
7392	do \
7393	{ \
7394	if (pOrgCtx->a_Field != pDebugCtx->a_Field) \
7395	{ \
7396	RTAssertMsg2Weak(" %8s differs - iem=%02x - rem=%02x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); \
7397	cDiffs++; \
7398	} \
7399	} while (0)
7400
7401	# define CHECK_SEL(a_Sel) \
7402	do \
7403	{ \
7404	CHECK_FIELD(a_Sel); \
7405	if ( pOrgCtx->a_Sel##Hid.Attr.u != pDebugCtx->a_Sel##Hid.Attr.u \
7406	&& (pOrgCtx->a_Sel##Hid.Attr.u \| X86_SEL_TYPE_ACCESSED) != pDebugCtx->a_Sel##Hid.Attr.u) \
7407	{ \
7408	RTAssertMsg2Weak(" %8sHid.Attr differs - iem=%02x - rem=%02x\n", #a_Sel, pDebugCtx->a_Sel##Hid.Attr.u, pOrgCtx->a_Sel##Hid.Attr.u); \
7409	cDiffs++; \
7410	} \
7411	CHECK_FIELD(a_Sel##Hid.u64Base); \
7412	CHECK_FIELD(a_Sel##Hid.u32Limit); \
7413	} while (0)
7414
7415	#if 1 /* The recompiler doesn't update these the intel way. */
7416	pOrgCtx->fpu.FOP = pDebugCtx->fpu.FOP;
7417	pOrgCtx->fpu.FPUIP = pDebugCtx->fpu.FPUIP;
7418	pOrgCtx->fpu.CS = pDebugCtx->fpu.CS;
7419	pOrgCtx->fpu.Rsrvd1 = pDebugCtx->fpu.Rsrvd1;
7420	pOrgCtx->fpu.FPUDP = pDebugCtx->fpu.FPUDP;
7421	pOrgCtx->fpu.DS = pDebugCtx->fpu.DS;
7422	pOrgCtx->fpu.Rsrvd2 = pDebugCtx->fpu.Rsrvd2;
7423	pOrgCtx->fpu.MXCSR_MASK = pDebugCtx->fpu.MXCSR_MASK; /* only for the time being - old snapshots here. */
7424	if ((pOrgCtx->fpu.FSW & X86_FSW_TOP_MASK) == (pDebugCtx->fpu.FSW & X86_FSW_TOP_MASK))
7425	pOrgCtx->fpu.FSW = pDebugCtx->fpu.FSW;
7426	#endif
7427	if (memcmp(&pOrgCtx->fpu, &pDebugCtx->fpu, sizeof(pDebugCtx->fpu)))
7428	{
7429	RTAssertMsg2Weak(" the FPU state differs\n");
7430	cDiffs++;
7431	CHECK_FIELD(fpu.FCW);
7432	CHECK_FIELD(fpu.FSW);
7433	CHECK_FIELD(fpu.FTW);
7434	CHECK_FIELD(fpu.FOP);
7435	CHECK_FIELD(fpu.FPUIP);
7436	CHECK_FIELD(fpu.CS);
7437	CHECK_FIELD(fpu.Rsrvd1);
7438	CHECK_FIELD(fpu.FPUDP);
7439	CHECK_FIELD(fpu.DS);
7440	CHECK_FIELD(fpu.Rsrvd2);
7441	CHECK_FIELD(fpu.MXCSR);
7442	CHECK_FIELD(fpu.MXCSR_MASK);
7443	CHECK_FIELD(fpu.aRegs[0].au64[0]); CHECK_FIELD(fpu.aRegs[0].au64[1]);
7444	CHECK_FIELD(fpu.aRegs[1].au64[0]); CHECK_FIELD(fpu.aRegs[1].au64[1]);
7445	CHECK_FIELD(fpu.aRegs[2].au64[0]); CHECK_FIELD(fpu.aRegs[2].au64[1]);
7446	CHECK_FIELD(fpu.aRegs[3].au64[0]); CHECK_FIELD(fpu.aRegs[3].au64[1]);
7447	CHECK_FIELD(fpu.aRegs[4].au64[0]); CHECK_FIELD(fpu.aRegs[4].au64[1]);
7448	CHECK_FIELD(fpu.aRegs[5].au64[0]); CHECK_FIELD(fpu.aRegs[5].au64[1]);
7449	CHECK_FIELD(fpu.aRegs[6].au64[0]); CHECK_FIELD(fpu.aRegs[6].au64[1]);
7450	CHECK_FIELD(fpu.aRegs[7].au64[0]); CHECK_FIELD(fpu.aRegs[7].au64[1]);
7451	CHECK_FIELD(fpu.aXMM[ 0].au64[0]); CHECK_FIELD(fpu.aXMM[ 0].au64[1]);
7452	CHECK_FIELD(fpu.aXMM[ 1].au64[0]); CHECK_FIELD(fpu.aXMM[ 1].au64[1]);
7453	CHECK_FIELD(fpu.aXMM[ 2].au64[0]); CHECK_FIELD(fpu.aXMM[ 2].au64[1]);
7454	CHECK_FIELD(fpu.aXMM[ 3].au64[0]); CHECK_FIELD(fpu.aXMM[ 3].au64[1]);
7455	CHECK_FIELD(fpu.aXMM[ 4].au64[0]); CHECK_FIELD(fpu.aXMM[ 4].au64[1]);
7456	CHECK_FIELD(fpu.aXMM[ 5].au64[0]); CHECK_FIELD(fpu.aXMM[ 5].au64[1]);
7457	CHECK_FIELD(fpu.aXMM[ 6].au64[0]); CHECK_FIELD(fpu.aXMM[ 6].au64[1]);
7458	CHECK_FIELD(fpu.aXMM[ 7].au64[0]); CHECK_FIELD(fpu.aXMM[ 7].au64[1]);
7459	CHECK_FIELD(fpu.aXMM[ 8].au64[0]); CHECK_FIELD(fpu.aXMM[ 8].au64[1]);
7460	CHECK_FIELD(fpu.aXMM[ 9].au64[0]); CHECK_FIELD(fpu.aXMM[ 9].au64[1]);
7461	CHECK_FIELD(fpu.aXMM[10].au64[0]); CHECK_FIELD(fpu.aXMM[10].au64[1]);
7462	CHECK_FIELD(fpu.aXMM[11].au64[0]); CHECK_FIELD(fpu.aXMM[11].au64[1]);
7463	CHECK_FIELD(fpu.aXMM[12].au64[0]); CHECK_FIELD(fpu.aXMM[12].au64[1]);
7464	CHECK_FIELD(fpu.aXMM[13].au64[0]); CHECK_FIELD(fpu.aXMM[13].au64[1]);
7465	CHECK_FIELD(fpu.aXMM[14].au64[0]); CHECK_FIELD(fpu.aXMM[14].au64[1]);
7466	CHECK_FIELD(fpu.aXMM[15].au64[0]); CHECK_FIELD(fpu.aXMM[15].au64[1]);
7467	for (unsigned i = 0; i < RT_ELEMENTS(pOrgCtx->fpu.au32RsrvdRest); i++)
7468	CHECK_FIELD(fpu.au32RsrvdRest[i]);
7469	}
7470	CHECK_FIELD(rip);
7471	uint32_t fFlagsMask = UINT32_MAX & ~pIemCpu->fUndefinedEFlags;
7472	if ((pOrgCtx->rflags.u & fFlagsMask) != (pDebugCtx->rflags.u & fFlagsMask))
7473	{
7474	RTAssertMsg2Weak(" rflags differs - iem=%08llx rem=%08llx\n", pDebugCtx->rflags.u, pOrgCtx->rflags.u);
7475	CHECK_BIT_FIELD(rflags.Bits.u1CF);
7476	CHECK_BIT_FIELD(rflags.Bits.u1Reserved0);
7477	CHECK_BIT_FIELD(rflags.Bits.u1PF);
7478	CHECK_BIT_FIELD(rflags.Bits.u1Reserved1);
7479	CHECK_BIT_FIELD(rflags.Bits.u1AF);
7480	CHECK_BIT_FIELD(rflags.Bits.u1Reserved2);
7481	CHECK_BIT_FIELD(rflags.Bits.u1ZF);
7482	CHECK_BIT_FIELD(rflags.Bits.u1SF);
7483	CHECK_BIT_FIELD(rflags.Bits.u1TF);
7484	CHECK_BIT_FIELD(rflags.Bits.u1IF);
7485	CHECK_BIT_FIELD(rflags.Bits.u1DF);
7486	CHECK_BIT_FIELD(rflags.Bits.u1OF);
7487	CHECK_BIT_FIELD(rflags.Bits.u2IOPL);
7488	CHECK_BIT_FIELD(rflags.Bits.u1NT);
7489	CHECK_BIT_FIELD(rflags.Bits.u1Reserved3);
7490	CHECK_BIT_FIELD(rflags.Bits.u1RF);
7491	CHECK_BIT_FIELD(rflags.Bits.u1VM);
7492	CHECK_BIT_FIELD(rflags.Bits.u1AC);
7493	CHECK_BIT_FIELD(rflags.Bits.u1VIF);
7494	CHECK_BIT_FIELD(rflags.Bits.u1VIP);
7495	CHECK_BIT_FIELD(rflags.Bits.u1ID);
7496	}
7497
7498	if (pIemCpu->cIOReads != 1 && !pIemCpu->fIgnoreRaxRdx)
7499	CHECK_FIELD(rax);
7500	CHECK_FIELD(rcx);
7501	if (!pIemCpu->fIgnoreRaxRdx)
7502	CHECK_FIELD(rdx);
7503	CHECK_FIELD(rbx);
7504	CHECK_FIELD(rsp);
7505	CHECK_FIELD(rbp);
7506	CHECK_FIELD(rsi);
7507	CHECK_FIELD(rdi);
7508	CHECK_FIELD(r8);
7509	CHECK_FIELD(r9);
7510	CHECK_FIELD(r10);
7511	CHECK_FIELD(r11);
7512	CHECK_FIELD(r12);
7513	CHECK_FIELD(r13);
7514	CHECK_SEL(cs);
7515	CHECK_SEL(ss);
7516	CHECK_SEL(ds);
7517	CHECK_SEL(es);
7518	CHECK_SEL(fs);
7519	CHECK_SEL(gs);
7520	CHECK_FIELD(cr0);
7521	CHECK_FIELD(cr2);
7522	CHECK_FIELD(cr3);
7523	CHECK_FIELD(cr4);
7524	CHECK_FIELD(dr[0]);
7525	CHECK_FIELD(dr[1]);
7526	CHECK_FIELD(dr[2]);
7527	CHECK_FIELD(dr[3]);
7528	CHECK_FIELD(dr[6]);
7529	if ((pOrgCtx->dr[7] & ~X86_DR7_MB1_MASK) != (pDebugCtx->dr[7] & ~X86_DR7_MB1_MASK)) /* REM 'mov drX,greg' bug.*/
7530	CHECK_FIELD(dr[7]);
7531	CHECK_FIELD(gdtr.cbGdt);
7532	CHECK_FIELD(gdtr.pGdt);
7533	CHECK_FIELD(idtr.cbIdt);
7534	CHECK_FIELD(idtr.pIdt);
7535	CHECK_FIELD(ldtr);
7536	CHECK_FIELD(ldtrHid.u64Base);
7537	CHECK_FIELD(ldtrHid.u32Limit);
7538	CHECK_FIELD(ldtrHid.Attr.u);
7539	CHECK_FIELD(tr);
7540	CHECK_FIELD(trHid.u64Base);
7541	CHECK_FIELD(trHid.u32Limit);
7542	CHECK_FIELD(trHid.Attr.u);
7543	CHECK_FIELD(SysEnter.cs);
7544	CHECK_FIELD(SysEnter.eip);
7545	CHECK_FIELD(SysEnter.esp);
7546	CHECK_FIELD(msrEFER);
7547	CHECK_FIELD(msrSTAR);
7548	CHECK_FIELD(msrPAT);
7549	CHECK_FIELD(msrLSTAR);
7550	CHECK_FIELD(msrCSTAR);
7551	CHECK_FIELD(msrSFMASK);
7552	CHECK_FIELD(msrKERNELGSBASE);
7553
7554	if (cDiffs != 0)
7555	{
7556	if (LogIs3Enabled())
7557	DBGFR3Info(pVM, "cpumguest", "verbose", NULL);
7558	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__);
7559	iemVerifyAssertMsg2(pIemCpu);
7560	RTAssertPanic();
7561	}
7562	# undef CHECK_FIELD
7563	# undef CHECK_BIT_FIELD
7564	}
7565
7566	/*
7567	* If the register state compared fine, check the verification event
7568	* records.
7569	*/
7570	if (cDiffs == 0)
7571	{
7572	/*
7573	* Compare verficiation event records.
7574	* - I/O port accesses should be a 1:1 match.
7575	*/
7576	PIEMVERIFYEVTREC pIemRec = pIemCpu->pIemEvtRecHead;
7577	PIEMVERIFYEVTREC pOtherRec = pIemCpu->pOtherEvtRecHead;
7578	while (pIemRec && pOtherRec)
7579	{
7580	/* Since we might miss RAM writes and reads, ignore reads and check
7581	that any written memory is the same extra ones. */
7582	while ( IEMVERIFYEVENT_IS_RAM(pIemRec->enmEvent)
7583	&& !IEMVERIFYEVENT_IS_RAM(pOtherRec->enmEvent)
7584	&& pIemRec->pNext)
7585	{
7586	if (pIemRec->enmEvent == IEMVERIFYEVENT_RAM_WRITE)
7587	iemVerifyWriteRecord(pIemCpu, pIemRec);
7588	pIemRec = pIemRec->pNext;
7589	}
7590
7591	/* Do the compare. */
7592	if (pIemRec->enmEvent != pOtherRec->enmEvent)
7593	{
7594	iemVerifyAssertRecords(pIemCpu, pIemRec, pOtherRec, "Type mismatches");
7595	break;
7596	}
7597	bool fEquals;
7598	switch (pIemRec->enmEvent)
7599	{
7600	case IEMVERIFYEVENT_IOPORT_READ:
7601	fEquals = pIemRec->u.IOPortRead.Port == pOtherRec->u.IOPortRead.Port
7602	&& pIemRec->u.IOPortRead.cbValue == pOtherRec->u.IOPortRead.cbValue;
7603	break;
7604	case IEMVERIFYEVENT_IOPORT_WRITE:
7605	fEquals = pIemRec->u.IOPortWrite.Port == pOtherRec->u.IOPortWrite.Port
7606	&& pIemRec->u.IOPortWrite.cbValue == pOtherRec->u.IOPortWrite.cbValue
7607	&& pIemRec->u.IOPortWrite.u32Value == pOtherRec->u.IOPortWrite.u32Value;
7608	break;
7609	case IEMVERIFYEVENT_RAM_READ:
7610	fEquals = pIemRec->u.RamRead.GCPhys == pOtherRec->u.RamRead.GCPhys
7611	&& pIemRec->u.RamRead.cb == pOtherRec->u.RamRead.cb;
7612	break;
7613	case IEMVERIFYEVENT_RAM_WRITE:
7614	fEquals = pIemRec->u.RamWrite.GCPhys == pOtherRec->u.RamWrite.GCPhys
7615	&& pIemRec->u.RamWrite.cb == pOtherRec->u.RamWrite.cb
7616	&& !memcmp(pIemRec->u.RamWrite.ab, pOtherRec->u.RamWrite.ab, pIemRec->u.RamWrite.cb);
7617	break;
7618	default:
7619	fEquals = false;
7620	break;
7621	}
7622	if (!fEquals)
7623	{
7624	iemVerifyAssertRecords(pIemCpu, pIemRec, pOtherRec, "Mismatch");
7625	break;
7626	}
7627
7628	/* advance */
7629	pIemRec = pIemRec->pNext;
7630	pOtherRec = pOtherRec->pNext;
7631	}
7632
7633	/* Ignore extra writes and reads. */
7634	while (pIemRec && IEMVERIFYEVENT_IS_RAM(pIemRec->enmEvent))
7635	{
7636	if (pIemRec->enmEvent == IEMVERIFYEVENT_RAM_WRITE)
7637	iemVerifyWriteRecord(pIemCpu, pIemRec);
7638	pIemRec = pIemRec->pNext;
7639	}
7640	if (pIemRec != NULL)
7641	iemVerifyAssertRecord(pIemCpu, pIemRec, "Extra IEM record!");
7642	else if (pOtherRec != NULL)
7643	iemVerifyAssertRecord(pIemCpu, pIemRec, "Extra Other record!");
7644	}
7645	pIemCpu->CTX_SUFF(pCtx) = pOrgCtx;
7646
7647	#if 0
7648	/*
7649	* HACK ALERT! You don't normally want to verify a whole boot sequence.
7650	*/
7651	if (pIemCpu->cInstructions == 1)
7652	RTLogFlags(NULL, "disabled");
7653	#endif
7654	}
7655
7656	#else /* !IEM_VERIFICATION_MODE \|\| !IN_RING3 */
7657
7658	/* stubs */
7659	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue)
7660	{
7661	NOREF(pIemCpu); NOREF(Port); NOREF(pu32Value); NOREF(cbValue);
7662	return VERR_INTERNAL_ERROR;
7663	}
7664
7665	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
7666	{
7667	NOREF(pIemCpu); NOREF(Port); NOREF(u32Value); NOREF(cbValue);
7668	return VERR_INTERNAL_ERROR;
7669	}
7670
7671	#endif /* !IEM_VERIFICATION_MODE \|\| !IN_RING3 */
7672
7673
7674	/**
7675	* Execute one instruction.
7676	*
7677	* @return Strict VBox status code.
7678	* @param pVCpu The current virtual CPU.
7679	*/
7680	VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPU pVCpu)
7681	{
7682	PIEMCPU pIemCpu = &pVCpu->iem.s;
7683
7684	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
7685	iemExecVerificationModeSetup(pIemCpu);
7686	#endif
7687	#ifdef LOG_ENABLED
7688	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7689	if (LogIs2Enabled())
7690	{
7691	char szInstr[256];
7692	uint32_t cbInstr = 0;
7693	DBGFR3DisasInstrEx(pVCpu->pVMR3, pVCpu->idCpu, 0, 0,
7694	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
7695	szInstr, sizeof(szInstr), &cbInstr);
7696
7697	Log3(("**** "
7698	" eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
7699	" eip=%08x esp=%08x ebp=%08x iopl=%d\n"
7700	" cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
7701	" fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
7702	" %s\n"
7703	,
7704	pCtx->eax, pCtx->ebx, pCtx->ecx, pCtx->edx, pCtx->esi, pCtx->edi,
7705	pCtx->eip, pCtx->esp, pCtx->ebp, pCtx->eflags.Bits.u2IOPL,
7706	(RTSEL)pCtx->cs, (RTSEL)pCtx->ss, (RTSEL)pCtx->ds, (RTSEL)pCtx->es,
7707	(RTSEL)pCtx->fs, (RTSEL)pCtx->gs, pCtx->eflags.u,
7708	pCtx->fpu.FSW, pCtx->fpu.FCW, pCtx->fpu.FTW, pCtx->fpu.MXCSR, pCtx->fpu.MXCSR_MASK,
7709	szInstr));
7710
7711	if (LogIs3Enabled())
7712	DBGFR3Info(pVCpu->pVMR3, "cpumguest", "verbose", NULL);
7713	}
7714	else
7715	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n",
7716	pCtx->cs, pCtx->rip, pCtx->ss, pCtx->rsp, pCtx->eflags.u));
7717	#endif
7718
7719	/*
7720	* Do the decoding and emulation.
7721	*/
7722	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu);
7723	if (rcStrict != VINF_SUCCESS)
7724	{
7725	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
7726	iemExecVerificationModeCheck(pIemCpu);
7727	#endif
7728	return rcStrict;
7729	}
7730
7731	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
7732	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
7733	if (rcStrict == VINF_SUCCESS)
7734	pIemCpu->cInstructions++;
7735	//#ifdef DEBUG
7736	// AssertMsg(pIemCpu->offOpcode == cbInstr \|\| rcStrict != VINF_SUCCESS, ("%u %u\n", pIemCpu->offOpcode, cbInstr));
7737	//#endif
7738
7739	/* Execute the next instruction as well if a cli, pop ss or
7740	mov ss, Gr has just completed successfully. */
7741	if ( rcStrict == VINF_SUCCESS
7742	&& VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
7743	&& EMGetInhibitInterruptsPC(pVCpu) == pIemCpu->CTX_SUFF(pCtx)->rip )
7744	{
7745	rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu);
7746	if (rcStrict == VINF_SUCCESS)
7747	{
7748	b; IEM_OPCODE_GET_NEXT_U8(&b);
7749	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
7750	if (rcStrict == VINF_SUCCESS)
7751	pIemCpu->cInstructions++;
7752	}
7753	EMSetInhibitInterruptsPC(pVCpu, UINT64_C(0x7777555533331111));
7754	}
7755
7756	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
7757	/*
7758	* Assert some sanity.
7759	*/
7760	iemExecVerificationModeCheck(pIemCpu);
7761	#endif
7762	if (rcStrict != VINF_SUCCESS)
7763	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
7764	pCtx->cs, pCtx->rip, pCtx->ss, pCtx->rsp, pCtx->eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
7765	return rcStrict;
7766	}
7767
7768
7769	/**
7770	* Injects a trap, fault, abort, software interrupt or external interrupt.
7771	*
7772	* The parameter list matches TRPMQueryTrapAll pretty closely.
7773	*
7774	* @returns Strict VBox status code.
7775	* @param pVCpu The current virtual CPU.
7776	* @param u8TrapNo The trap number.
7777	* @param enmType What type is it (trap/fault/abort), software
7778	* interrupt or hardware interrupt.
7779	* @param uErrCode The error code if applicable.
7780	* @param uCr2 The CR2 value if applicable.
7781	*/
7782	VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPU pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2)
7783	{
7784	iemInitDecoder(&pVCpu->iem.s);
7785
7786	uint32_t fFlags;
7787	switch (enmType)
7788	{
7789	case TRPM_HARDWARE_INT:
7790	LogFlow(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
7791	fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7792	uErrCode = uCr2 = 0;
7793	break;
7794
7795	case TRPM_SOFTWARE_INT:
7796	LogFlow(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
7797	fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7798	uErrCode = uCr2 = 0;
7799	break;
7800
7801	case TRPM_TRAP:
7802	LogFlow(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
7803	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7804	if (u8TrapNo == X86_XCPT_PF)
7805	fFlags \|= IEM_XCPT_FLAGS_CR2;
7806	switch (u8TrapNo)
7807	{
7808	case X86_XCPT_DF:
7809	case X86_XCPT_TS:
7810	case X86_XCPT_NP:
7811	case X86_XCPT_SS:
7812	case X86_XCPT_PF:
7813	case X86_XCPT_AC:
7814	fFlags \|= IEM_XCPT_FLAGS_ERR;
7815	break;
7816	}
7817	break;
7818
7819	IEM_NOT_REACHED_DEFAULT_CASE_RET();
7820	}
7821
7822	return iemRaiseXcptOrInt(&pVCpu->iem.s, 0, u8TrapNo, fFlags, uErrCode, uCr2);
7823	}
7824

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 40255

Download in other formats: