IEMMc.h@ 106695

Last change on this file since 106695 was 106427, checked in by vboxsync, 6 weeks ago
VMM/IEM: Reduced the paramters for iemNativeEmitRetn. bugref:10720
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 182.4 KB

Line
1	/* $Id: IEMMc.h 106427 2024-10-17 10:59:12Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29	#define VMM_INCLUDED_SRC_include_IEMMc_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @name "Microcode" macros.
36	*
37	* The idea is that we should be able to use the same code to interpret
38	* instructions as well as recompiler instructions. Thus this obfuscation.
39	*
40	* @{
41	*/
42
43	#define IEM_MC_BEGIN(a_fMcFlags, a_fCImplFlags) {
44	#define IEM_MC_END() }
45
46	/** Internal macro. */
47	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48	do \
49	{ \
50	VBOXSTRICTRC rcStrict2 = a_Expr; \
51	if (rcStrict2 == VINF_SUCCESS) \
52	{ /* likely */ } \
53	else \
54	return rcStrict2; \
55	} while (0)
56
57
58	/** Dummy MC that prevents native recompilation. */
59	#define IEM_MC_NO_NATIVE_RECOMPILE() ((void)0)
60
61	/** Advances RIP, finishes the instruction and returns.
62	* This may include raising debug exceptions and such. */
63	#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
64	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
65	#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
66	return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
67	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
68	* @note only usable in 16-bit op size mode. */
69	#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
70	return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
71	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72	#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
73	return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
74	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
75	#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) \
76	return iemRegRipJumpU16AndFinishClearingRF((pVCpu), (a_u16NewIP), IEM_GET_INSTR_LEN(pVCpu))
77	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
78	#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) \
79	return iemRegRipJumpU32AndFinishClearingRF((pVCpu), (a_u32NewIP), IEM_GET_INSTR_LEN(pVCpu))
80	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
81	#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) \
82	return iemRegRipJumpU64AndFinishClearingRF((pVCpu), (a_u64NewIP), IEM_GET_INSTR_LEN(pVCpu))
83
84	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
85	* @note only usable in 16-bit op size mode. */
86	#define IEM_MC_REL_CALL_S16_AND_FINISH(a_i16) \
87	return iemRegRipRelativeCallS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
88	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
89	#define IEM_MC_REL_CALL_S32_AND_FINISH(a_i32) \
90	return iemRegEip32RelativeCallS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32))
91	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
92	#define IEM_MC_REL_CALL_S64_AND_FINISH(a_i64) \
93	return iemRegRip64RelativeCallS64AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i64))
94	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
95	#define IEM_MC_IND_CALL_U16_AND_FINISH(a_u16NewIP) \
96	return iemRegIp16IndirectCallU16AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u16NewIP))
97	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
98	#define IEM_MC_IND_CALL_U32_AND_FINISH(a_u32NewIP) \
99	return iemRegEip32IndirectCallU32AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u32NewIP))
100	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
101	#define IEM_MC_IND_CALL_U64_AND_FINISH(a_u64NewIP) \
102	return iemRegRip64IndirectCallU64AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u64NewIP))
103
104
105	/** Fetches the near return address from the stack, sets RIP and RSP (may trigger
106	* \#GP or \#SS), finishes the instruction and returns. */
107	#define IEM_MC_RETN_AND_FINISH(a_cbPopArgs) \
108	return iemRegRipNearReturnAndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_cbPopArgs), pVCpu->iem.s.enmEffOpSize)
109
110
111	#define IEM_MC_RAISE_DIVIDE_ERROR_IF_LOCAL_IS_ZERO(a_uVar) \
112	do { \
113	if (RT_LIKELY((a_uVar) != 0)) \
114	{ /* probable */ } \
115	else return iemRaiseDivideError(pVCpu); \
116	} while (0)
117	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
118	do { \
119	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)))) \
120	{ /* probable */ } \
121	else return iemRaiseDeviceNotAvailable(pVCpu); \
122	} while (0)
123	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
124	do { \
125	if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)))) \
126	{ /* probable */ } \
127	else return iemRaiseDeviceNotAvailable(pVCpu); \
128	} while (0)
129	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
130	do { \
131	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
132	{ /* probable */ } \
133	else return iemRaiseMathFault(pVCpu); \
134	} while (0)
135	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
136	do { \
137	/* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
138	be reduced to a single compare branch in the more probably code path. */ \
139	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) \
140	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
141	\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
142	== (XSAVE_C_YMM \| XSAVE_C_SSE \| X86_CR4_OSXSAVE))) \
143	{ /* probable */ } \
144	else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
145	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
146	return iemRaiseUndefinedOpcode(pVCpu); \
147	else \
148	return iemRaiseDeviceNotAvailable(pVCpu); \
149	} while (0)
150	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR4_OSXSAVE));
151	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR0_TS));
152	AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
153	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
154	do { \
155	/* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
156	single compare branch in the more probable code path. */ \
157	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
158	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
159	== X86_CR4_OSFXSR)) \
160	{ /* likely */ } \
161	else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
162	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
163	return iemRaiseUndefinedOpcode(pVCpu); \
164	else \
165	return iemRaiseDeviceNotAvailable(pVCpu); \
166	} while (0)
167	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_CR4_OSFXSR));
168	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
169	do { \
170	/* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
171	single compare branch in the more probable code path. */ \
172	if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
173	\| (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
174	{ /* probable */ } \
175	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
176	return iemRaiseUndefinedOpcode(pVCpu); \
177	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
178	return iemRaiseDeviceNotAvailable(pVCpu); \
179	else \
180	return iemRaiseMathFault(pVCpu); \
181	} while (0)
182	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_FSW_ES));
183	/** @todo recomp: this one is slightly problematic as the recompiler doesn't
184	* count the CPL into the TB key. However it is safe enough for now, as
185	* it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
186	* emitted for it. */
187	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
188	do { \
189	if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
190	else return iemRaiseGeneralProtectionFault0(pVCpu); \
191	} while (0)
192	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
193	do { \
194	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
195	else return iemRaiseGeneralProtectionFault0(pVCpu); \
196	} while (0)
197	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
198	do { \
199	if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) \| IEM_GET_CPU_MODE(pVCpu)) \
200	== (X86_CR4_FSGSBASE \| IEMMODE_64BIT))) \
201	{ /* probable */ } \
202	else return iemRaiseUndefinedOpcode(pVCpu); \
203	} while (0)
204	AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
205	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
206	do { \
207	if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
208	else return iemRaiseGeneralProtectionFault0(pVCpu); \
209	} while (0)
210
211
212	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
213	#define IEM_MC_LOCAL_ASSIGN(a_Type, a_Name, a_Value) a_Type a_Name = (a_Value)
214	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
215	#define IEM_MC_NOREF(a_Name) RT_NOREF_PV(a_Name) /* NOP/liveness hack */
216	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
217	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
218	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
219	/** @note IEMAllInstPython.py duplicates the expansion. */
220	#define IEM_MC_ARG_EFLAGS(a_Name, a_iArg) uint32_t const a_Name = pVCpu->cpum.GstCtx.eflags.u
221	/** @note IEMAllInstPython.py duplicates the expansion. */
222	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
223	uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u; \
224	uint32_t *a_pName = &a_Name
225	/** @note IEMAllInstPython.py duplicates the expansion. */
226	#define IEM_MC_LOCAL_EFLAGS(a_Name) uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u
227	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
228	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
229	#define IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) do { \
230	AssertMsg((pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) == ((a_EFlags) & ~(a_fEflOutput)), \
231	("eflags.u=%#x (%#x) vs %s=%#x (%#x) - diff %#x (a_fEflOutput=%#x)\n", \
232	pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput), pVCpu->cpum.GstCtx.eflags.u, #a_EFlags, \
233	(a_EFlags) & ~(a_fEflOutput), (a_EFlags), \
234	(pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) ^ ((a_EFlags) & ~(a_fEflOutput)), a_fEflOutput)); \
235	pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); \
236	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); \
237	} while (0)
238	#define IEM_MC_COMMIT_EFLAGS_OPT(a_EFlags) IEM_MC_COMMIT_EFLAGS(a_EFlags)
239	#define IEM_MC_COMMIT_EFLAGS_OPT_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput)
240
241	/** ASSUMES the source variable not used after this statement. */
242	#define IEM_MC_ASSIGN_TO_SMALLER(a_VarDst, a_VarSrcEol) (a_VarDst) = (a_VarSrcEol)
243
244	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
245	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
246	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
247	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
248	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
249	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
250	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
251	#define IEM_MC_FETCH_GREG_I16(a_i16Dst, a_iGReg) (a_i16Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
252	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
253	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
254	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
255	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
256	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
257	#define IEM_MC_FETCH_GREG_I32(a_i32Dst, a_iGReg) (a_i32Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
258	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
259	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
260	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
261	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
262	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
263	#define IEM_MC_FETCH_GREG_PAIR_U32(a_u64Dst, a_iGRegLo, a_iGRegHi) do { \
264	(a_u64Dst).s.Lo = iemGRegFetchU32(pVCpu, (a_iGRegLo)); \
265	(a_u64Dst).s.Hi = iemGRegFetchU32(pVCpu, (a_iGRegHi)); \
266	} while(0)
267	#define IEM_MC_FETCH_GREG_PAIR_U64(a_u128Dst, a_iGRegLo, a_iGRegHi) do { \
268	(a_u128Dst).s.Lo = iemGRegFetchU64(pVCpu, (a_iGRegLo)); \
269	(a_u128Dst).s.Hi = iemGRegFetchU64(pVCpu, (a_iGRegHi)); \
270	} while(0)
271	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
272	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
273	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
274	} while (0)
275	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
276	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
277	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
278	} while (0)
279	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
280	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
281	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
282	} while (0)
283	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
284	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
285	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
286	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
287	} while (0)
288	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
289	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
290	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
291	} while (0)
292	/** @note Not for IOPL or IF testing or modification. */
293	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
294	#define IEM_MC_FETCH_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_FETCH_EFLAGS(a_EFlags)
295	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u /* (only LAHF) */
296	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
297	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
298
299	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
300	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
301	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
302	#define IEM_MC_STORE_GREG_I32(a_iGReg, a_i32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_i32Value) / clear high bits. */
303	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
304	#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
305	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
306	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
307	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
308	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
309	#define IEM_MC_STORE_GREG_PAIR_U32(a_iGRegLo, a_iGRegHi, a_u64Value) do { \
310	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint32_t)(a_u64Value).s.Lo; \
311	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint32_t)(a_u64Value).s.Hi; \
312	} while(0)
313	#define IEM_MC_STORE_GREG_PAIR_U64(a_iGRegLo, a_iGRegHi, a_u128Value) do { \
314	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint64_t)(a_u128Value).s.Lo; \
315	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint64_t)(a_u128Value).s.Hi; \
316	} while(0)
317	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
318
319	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
320	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
321	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
322	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
323	} while (0)
324	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
325	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
326	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
327	} while (0)
328	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
329	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
330
331
332	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
333	#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
334	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
335	#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
336	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
337	* Use IEM_MC_CLEAR_HIGH_GREG_U64! */
338	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
339	#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
340	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
341	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
342	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
343	#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
344	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
345	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
346	/** @note Not for IOPL or IF testing or modification.
347	* @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
348	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
349	#define IEM_MC_REF_EFLAGS_EX(a_pEFlags, a_fEflInput, a_fEflOutput) IEM_MC_REF_EFLAGS(a_pEFlags)
350
351	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
352	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
353	do { \
354	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
355	*pu32Reg += (a_u32Value); \
356	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
357	} while (0)
358	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
359
360	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8Const) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u8Const)
361	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
362	do { \
363	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
364	*pu32Reg -= (a_u8Const); \
365	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
366	} while (0)
367	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u8Const)
368	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
369
370	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
371	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
372	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
373	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
374	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
375	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
376	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
377
378	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
379	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
380	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
381	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
382
383	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
384	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
385	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
386
387	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
388	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
389	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
390
391	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
392	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
393	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
394
395	#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
396
397	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
398	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
399	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
400
401	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
402
403	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
404
405	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
406	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
407	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
408	do { \
409	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
410	*pu32Reg &= (a_u32Value); \
411	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
412	} while (0)
413	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
414
415	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
416	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
417	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
418	do { \
419	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
420	*pu32Reg \|= (a_u32Value); \
421	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
422	} while (0)
423	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
424
425	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
426	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
427	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
428
429	/** @note Not for IOPL or IF modification. */
430	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
431	/** @note Not for IOPL or IF modification. */
432	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
433	/** @note Not for IOPL or IF modification. */
434	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
435
436	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
437
438	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
439	#define IEM_MC_FPU_TO_MMX_MODE() do { \
440	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
441	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
442	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
443	} while (0)
444
445	/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
446	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
447	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
448	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
449	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
450	} while (0)
451
452	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
453	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
454	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg, a_iDWord) \
455	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[a_iDWord]; } while (0)
456	#define IEM_MC_FETCH_MREG_U16(a_u16Value, a_iMReg, a_iWord) \
457	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[a_iWord]; } while (0)
458	#define IEM_MC_FETCH_MREG_U8(a_u8Value, a_iMReg, a_iByte) \
459	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[a_iByte]; } while (0)
460	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) \
461	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
462	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
463	} while (0)
464	#define IEM_MC_STORE_MREG_U32(a_iMReg, a_iDword, a_u32Value) \
465	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[(a_iDword)] = (a_u32Value); \
466	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
467	} while (0)
468	#define IEM_MC_STORE_MREG_U16(a_iMReg, a_iWord, a_u16Value) \
469	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[(a_iWord)] = (a_u16Value); \
470	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
471	} while (0)
472	#define IEM_MC_STORE_MREG_U8(a_iMReg, a_iByte, a_u8Value) \
473	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[(a_iByte)] = (a_u8Value); \
474	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
475	} while (0)
476	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) \
477	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
478	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
479	} while (0)
480	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
481	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
482	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
483	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
484	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
485	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
486	#define IEM_MC_MODIFIED_MREG(a_iMReg) \
487	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
488	#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
489	do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
490
491	#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
492	do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
493	if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
494	if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
495	if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
496	} while (0)
497	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
498	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
499	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
500	} while (0)
501	#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
502	do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
503	(a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
504	} while (0)
505	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
506	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
507	#define IEM_MC_FETCH_XREG_R64(a_r64Value, a_iXReg, a_iQWord) \
508	do { (a_r64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[(a_iQWord)]; } while (0)
509	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
510	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
511	#define IEM_MC_FETCH_XREG_R32(a_r32Value, a_iXReg, a_iDWord) \
512	do { (a_r32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[(a_iDWord)]; } while (0)
513	#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
514	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
515	#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
516	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)]; } while (0)
517	#define IEM_MC_FETCH_XREG_PAIR_U128(a_Dst, a_iXReg1, a_iXReg2) \
518	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
519	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
520	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
521	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
522	} while (0)
523	#define IEM_MC_FETCH_XREG_PAIR_XMM(a_Dst, a_iXReg1, a_iXReg2) \
524	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
525	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
526	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
527	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
528	} while (0)
529	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iXReg2) \
530	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
531	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
532	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
533	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
534	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
535	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
536	} while (0)
537	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iXReg2) \
538	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
539	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
540	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
541	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
542	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
543	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
544	} while (0)
545	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
546	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
547	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
548	} while (0)
549	#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
550	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
551	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
552	} while (0)
553	#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
554	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
555	#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
556	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
557	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
558	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
559	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
560	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
561	#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
562	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)] = (a_u16Value); } while (0)
563	#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
564	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)] = (a_u8Value); } while (0)
565
566	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
567	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
568	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
569	} while (0)
570
571	#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
572	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
573	#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
574	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
575	#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
576	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
577	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
578	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
579	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
580	} while (0)
581
582	#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
583	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
584	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
585	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
586	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
587	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
588	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
589	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
590	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
591	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
592	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
593	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
594	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
595	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
596	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
597	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
598	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
599	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
600	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
601	} while (0)
602	#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
603	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
604	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
605	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
606	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
607	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
608	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
609	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
610	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
611	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
612	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
613	} while (0)
614	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
615	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
616	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
617	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
618	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
619	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
620	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
621	} while (0)
622	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
623	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
624	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
625	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
626	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
627	} while (0)
628
629	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
630	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
631	#define IEM_MC_REF_XREG_XMM(a_pXmmDst, a_iXReg) \
632	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
633	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
634	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
635	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
636	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
637	#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
638	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
639	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
640	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
641	#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
642	(a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
643	#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
644	(a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
645	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
646	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
647	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
648	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
649	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
650	} while (0)
651
652	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
653	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
654	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
655	} while (0)
656	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc, a_iQWord) \
657	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
658	if ((a_iQWord) < 2) \
659	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[(a_iQWord)]; \
660	else \
661	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[(a_iQWord) - 2]; \
662	} while (0)
663	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc, a_iDQword) \
664	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
665	if ((a_iDQword) == 0) \
666	{ \
667	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[0]; \
668	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[1]; \
669	} \
670	else \
671	{ \
672	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[0]; \
673	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[1]; \
674	} \
675	} while (0)
676	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
677	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
678	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
679	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
680	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
681	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
682	} while (0)
683	#define IEM_MC_FETCH_YREG_YMM(a_uYmmDst, a_iYRegSrc) \
684	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
685	(a_uYmmDst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
686	(a_uYmmDst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
687	(a_uYmmDst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
688	(a_uYmmDst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
689	} while (0)
690	#define IEM_MC_FETCH_YREG_PAIR_YMM(a_uYmmDst, a_iYRegSrc1, a_iYRegSrc2) \
691	do { uintptr_t const iYRegSrc1Tmp = (a_iYRegSrc1); \
692	uintptr_t const iYRegSrc2Tmp = (a_iYRegSrc2); \
693	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc1Tmp].au64[0]; \
694	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc1Tmp].au64[1]; \
695	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc1Tmp].au64[0]; \
696	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc1Tmp].au64[1]; \
697	(a_uYmmDst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc2Tmp].au64[0]; \
698	(a_uYmmDst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc2Tmp].au64[1]; \
699	(a_uYmmDst).uSrc2.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc2Tmp].au64[0]; \
700	(a_uYmmDst).uSrc2.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc2Tmp].au64[1]; \
701	} while (0)
702
703	#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
704	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
705	if ((a_iDQword) == 0) \
706	{ \
707	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
708	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
709	} \
710	else \
711	{ \
712	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
713	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
714	} \
715	} while (0)
716
717	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
718	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
719	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
720	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
721	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
722	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
723	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
724	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
725	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
726	} while (0)
727	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
728	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
729	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
730	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
731	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
732	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
733	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
734	} while (0)
735	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
736	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
737	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
738	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
739	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
740	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
741	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
742	} while (0)
743	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
744	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
745	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
746	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
747	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
748	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
749	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
750	} while (0)
751	#define IEM_MC_STORE_YREG_YMM_ZX_VLMAX(a_iYRegDst, a_uYmmSrc) \
752	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
753	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_uYmmSrc).au64[0]; \
754	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_uYmmSrc).au64[1]; \
755	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_uYmmSrc).au64[2]; \
756	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_uYmmSrc).au64[3]; \
757	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
758	} while (0)
759	#define IEM_MC_STORE_YREG_U32_U256(a_iYRegDst, a_iDwDst, a_u256Value, a_iDwSrc) \
760	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
761	if ((a_iDwDst) < 4) \
762	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au32[(a_iDwDst)] = (a_u256Value).au32[(a_iDwSrc)]; \
763	else \
764	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au32[(a_iDwDst) - 4] = (a_u256Value).au32[(a_iDwSrc)]; \
765	} while (0)
766	#define IEM_MC_STORE_YREG_U64_U256(a_iYRegDst, a_iQwDst, a_u256Value, a_iQwSrc) \
767	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
768	if ((a_iQwDst) < 2) \
769	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQwDst)] = (a_u256Value).au64[(a_iQwSrc)]; \
770	else \
771	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQwDst) - 2] = (a_u256Value).au64[(a_iQwSrc)]; \
772	} while (0)
773	#define IEM_MC_STORE_YREG_U64(a_iYRegDst, a_iQword, a_u64Value) \
774	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
775	if ((a_iQword) < 2) \
776	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQword)] = (a_u64Value); \
777	else \
778	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQword) - 2] = (a_u64Value); \
779	} while (0)
780
781	#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
782	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
783	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
784	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
785	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
786	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
787	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
788	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
789	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
790	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
791	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
792	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
793	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
794	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
795	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
796	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
797	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
798	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
799	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
800	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
801	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
802	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
803	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
804	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
805	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
806	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
807	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
808	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
809	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
810	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
811	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
812	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
813	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
814	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
815	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
816	} while (0)
817	#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
818	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
819	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
820	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
821	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
822	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
823	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
824	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
825	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
826	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
827	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
828	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
829	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
830	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
831	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
832	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
833	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
834	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
835	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
836	} while (0)
837	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
838	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
839	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
840	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
841	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
842	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
843	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
844	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
845	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
846	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
847	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
848	} while (0)
849	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
850	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
851	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
852	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
853	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
854	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
855	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
856	} while (0)
857	#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
858	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
859	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
860	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
861	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
862	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
863	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
864	} while (0)
865
866	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
867	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
868	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
869	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
870	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
871	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
872	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
873	do { uintptr_t const iYRegTmp = (a_iYReg); \
874	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
875	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
876	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
877	} while (0)
878
879	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
880	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
881	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
882	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
883	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
884	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
885	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
886	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
887	} while (0)
888	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
889	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
890	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
891	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
892	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
893	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
894	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
895	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
896	} while (0)
897	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
898	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
899	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
900	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
901	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
902	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
903	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
904	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
905	} while (0)
906
907	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
908	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
909	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
910	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
911	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
912	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
913	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
914	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
915	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
916	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
917	} while (0)
918	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
919	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
920	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
921	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
922	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
923	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
924	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
925	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
926	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
927	} while (0)
928	#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
929	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
930	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
931	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
932	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
933	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
934	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
935	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
936	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
937	} while (0)
938	#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
939	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
940	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
941	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
942	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
943	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
944	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
945	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
946	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
947	} while (0)
948	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
949	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
950	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
951	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
952	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
953	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
954	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
955	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
956	} while (0)
957	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
958	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
959	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
960	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
961	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
962	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
963	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
964	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
965	} while (0)
966
967	#define IEM_MC_CLEAR_ZREG_256_UP(a_iYReg) \
968	do { IEM_MC_INT_CLEAR_ZMM_256_UP(a_iYReg); } while (0)
969
970	#ifndef IEM_WITH_SETJMP
971	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
972	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
973	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
974	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
975	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
976	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
977	#else
978	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
979	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
980	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
981	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
982	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
983	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
984
985	# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
986	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
987	# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
988	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
989	# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
990	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
991	#endif
992
993	#ifndef IEM_WITH_SETJMP
994	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
995	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
996	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
997	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
998	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
999	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
1000	# define IEM_MC_FETCH_MEM_I16_DISP(a_i16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1001	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1002	#else
1003	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1004	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1005	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1006	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1007	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
1008	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1009	# define IEM_MC_FETCH_MEM_I16_DISP(a_i16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1010	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1011
1012	# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
1013	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1014	# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
1015	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1016	# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
1017	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1018	# define IEM_MC_FETCH_MEM_FLAT_I16_DISP(a_i16Dst, a_GCPtrMem, a_offDisp) \
1019	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1020	#endif
1021
1022	#ifndef IEM_WITH_SETJMP
1023	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1024	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
1025	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1026	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1027	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
1028	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
1029	# define IEM_MC_FETCH_MEM_I32_DISP(a_i32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1030	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1031	#else
1032	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1033	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1034	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1035	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1036	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
1037	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1038	# define IEM_MC_FETCH_MEM_I32_DISP(a_i32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1039	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1040
1041	# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
1042	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1043	# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
1044	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1045	# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
1046	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1047	# define IEM_MC_FETCH_MEM_FLAT_I32_DISP(a_i32Dst, a_GCPtrMem, a_offDisp) \
1048	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1049	#endif
1050
1051	#ifndef IEM_WITH_SETJMP
1052	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1053	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
1054	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1055	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1056	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1057	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
1058	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1059	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
1060	#else
1061	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1062	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1063	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1064	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1065	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1066	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1067	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1068	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1069
1070	# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
1071	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1072	# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
1073	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1074	# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
1075	((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
1076	# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
1077	((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1078	#endif
1079
1080	#ifndef IEM_WITH_SETJMP
1081	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1082	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
1083	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1084	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
1085	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1086	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
1087	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1088	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
1089	#else
1090	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1091	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1092	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1093	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1094	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1095	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
1096	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1097	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
1098
1099	# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
1100	((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1101	# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
1102	((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1103	# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
1104	iemMemFlatFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_GCPtrMem))
1105	# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
1106	iemMemFlatFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_GCPtrMem))
1107	#endif
1108
1109	#ifndef IEM_WITH_SETJMP
1110	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1111	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1112	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1113	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1114	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1115	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1116
1117	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1118	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1119	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1120	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1121
1122	# define IEM_MC_FETCH_MEM_U128_NO_AC_AND_XREG_U128(a_u128Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1123	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1124	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1125	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1126	} while (0)
1127
1128	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1129	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2))); \
1130	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1131	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1132	} while (0)
1133
1134	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1135	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1136	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1137	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1138	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1139	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1140	} while (0)
1141
1142	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1143	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1144	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1145	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1146	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1147	} while (0)
1148
1149	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1150	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1151	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1152	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1153	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1154	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1155	} while (0)
1156	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1157	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1158	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1159	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1160	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1161	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1162	} while (0)
1163
1164	#else
1165	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1166	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1167	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1168	iemMemFetchDataU128NoAcJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1169	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1170	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1171
1172	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
1173	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1174	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1175	iemMemFetchDataU128NoAcJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1176	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1177	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1178
1179	# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
1180	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1181	# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
1182	iemMemFlatFetchDataU128NoAcJmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1183	# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
1184	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1185
1186	# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
1187	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1188	# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
1189	iemMemFlatFetchDataU128NoAcJmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1190	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
1191	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1192
1193	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1194	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1195	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1196	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1197	} while (0)
1198	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1199	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1200	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1201	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1202	} while (0)
1203
1204	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1205	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2)); \
1206	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1207	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1208	} while (0)
1209	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1210	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_GCPtrMem2)); \
1211	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1212	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1213	} while (0)
1214
1215	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1216	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1217	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1218	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1219	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1220	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1221	} while (0)
1222	# define IEM_MC_FETCH_MEM_FLAT_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_GCPtrMem2) do { \
1223	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1224	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1225	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem2)); \
1226	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1227	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1228	} while (0)
1229
1230	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1231	(a_Dst).uSrc2.uXmm.au64[!(a_iQWord2)] = 0; \
1232	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1233	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1234	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1235	} while (0)
1236	# define IEM_MC_FETCH_MEM_FLAT_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_GCPtrMem2) do { \
1237	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1238	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem2)); \
1239	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1240	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1241	} while (0)
1242
1243
1244	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1245	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1246	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1247	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1248	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1249	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1250	} while (0)
1251	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1252	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1253	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1254	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1255	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1256	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1257	} while (0)
1258
1259	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1260	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1261	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1262	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1263	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1264	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1265	} while (0)
1266	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1267	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1268	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1269	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1270	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1271	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1272	} while (0)
1273
1274	#endif
1275
1276	#ifndef IEM_WITH_SETJMP
1277	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1278	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1279	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1280	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1281	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1282	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1283
1284	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1285	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1286	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1287	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1288	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1289	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1290
1291	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_iSeg2, a_GCPtrMem2) do { \
1292	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1293	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_iSeg2), (a_GCPtrMem2))); \
1294	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1295	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1296	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1297	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1298	} while (0)
1299
1300	#else
1301	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1302	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1303	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1304	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1305	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1306	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1307
1308	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1309	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1310	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1311	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1312	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1313	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1314
1315	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_iSeg2, a_GCPtrMem2) do { \
1316	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1317	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_iSeg2), (a_GCPtrMem2)); \
1318	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1319	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1320	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1321	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1322	} while (0)
1323
1324	# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1325	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1326	# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1327	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1328	# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1329	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1330
1331	# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1332	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1333	# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1334	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1335	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1336	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1337
1338	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_GCPtrMem2) do { \
1339	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1340	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_GCPtrMem2)); \
1341	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1342	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1343	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1344	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1345	} while (0)
1346
1347	#endif
1348
1349
1350
1351	#ifndef IEM_WITH_SETJMP
1352	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1353	do { \
1354	uint8_t u8Tmp; \
1355	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1356	(a_u16Dst) = u8Tmp; \
1357	} while (0)
1358	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1359	do { \
1360	uint8_t u8Tmp; \
1361	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1362	(a_u32Dst) = u8Tmp; \
1363	} while (0)
1364	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1365	do { \
1366	uint8_t u8Tmp; \
1367	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1368	(a_u64Dst) = u8Tmp; \
1369	} while (0)
1370	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1371	do { \
1372	uint16_t u16Tmp; \
1373	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1374	(a_u32Dst) = u16Tmp; \
1375	} while (0)
1376	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1377	do { \
1378	uint16_t u16Tmp; \
1379	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1380	(a_u64Dst) = u16Tmp; \
1381	} while (0)
1382	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1383	do { \
1384	uint32_t u32Tmp; \
1385	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1386	(a_u64Dst) = u32Tmp; \
1387	} while (0)
1388	#else /* IEM_WITH_SETJMP */
1389	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1390	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1391	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1392	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1393	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1394	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1395	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1396	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1397	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1398	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1399	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1400	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1401
1402	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1403	((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1404	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1405	((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1406	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1407	((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1408	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1409	((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1410	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1411	((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1412	# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1413	((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1414	#endif /* IEM_WITH_SETJMP */
1415
1416	#ifndef IEM_WITH_SETJMP
1417	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1418	do { \
1419	uint8_t u8Tmp; \
1420	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1421	(a_u16Dst) = (int8_t)u8Tmp; \
1422	} while (0)
1423	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1424	do { \
1425	uint8_t u8Tmp; \
1426	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1427	(a_u32Dst) = (int8_t)u8Tmp; \
1428	} while (0)
1429	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1430	do { \
1431	uint8_t u8Tmp; \
1432	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1433	(a_u64Dst) = (int8_t)u8Tmp; \
1434	} while (0)
1435	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1436	do { \
1437	uint16_t u16Tmp; \
1438	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1439	(a_u32Dst) = (int16_t)u16Tmp; \
1440	} while (0)
1441	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1442	do { \
1443	uint16_t u16Tmp; \
1444	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1445	(a_u64Dst) = (int16_t)u16Tmp; \
1446	} while (0)
1447	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1448	do { \
1449	uint32_t u32Tmp; \
1450	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1451	(a_u64Dst) = (int32_t)u32Tmp; \
1452	} while (0)
1453	#else /* IEM_WITH_SETJMP */
1454	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1455	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1456	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1457	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1458	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1459	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1460	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1461	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1462	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1463	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1464	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1465	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1466
1467	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1468	((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1469	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1470	((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1471	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1472	((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1473	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1474	((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1475	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1476	((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1477	# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1478	((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1479	#endif /* IEM_WITH_SETJMP */
1480
1481	#ifndef IEM_WITH_SETJMP
1482	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1483	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1484	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1485	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1486	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1487	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1488	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1489	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1490	#else
1491	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1492	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1493	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1494	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1495	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1496	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1497	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1498	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1499
1500	# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1501	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1502	# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1503	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1504	# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1505	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1506	# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1507	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1508	#endif
1509
1510	#ifndef IEM_WITH_SETJMP
1511	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1512	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1513	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1514	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1515	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1516	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1517	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1518	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1519	#else
1520	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1521	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1522	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1523	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1524	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1525	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1526	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1527	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1528
1529	# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1530	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1531	# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1532	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1533	# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1534	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1535	# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1536	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1537	#endif
1538
1539	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1540	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1541	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1542	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1543	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1544	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1545	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1546	do { \
1547	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1548	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1549	} while (0)
1550	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1551	do { \
1552	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1553	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1554	} while (0)
1555
1556	#ifndef IEM_WITH_SETJMP
1557	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1558	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1559	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1560	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1561	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1562	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1563	#else
1564	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1565	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1566	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1567	iemMemStoreDataU128NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1568	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1569	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1570
1571	# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1572	iemMemFlatStoreDataU128Jmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1573	# define IEM_MC_STORE_MEM_FLAT_U128_NO_AC(a_GCPtrMem, a_u128Value) \
1574	iemMemFlatStoreDataU128NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1575	# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1576	iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1577	#endif
1578
1579	#ifndef IEM_WITH_SETJMP
1580	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1581	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1582	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1583	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1584	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1585	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1586	#else
1587	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1588	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1589	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1590	iemMemStoreDataU256NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1591	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1592	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1593
1594	# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1595	iemMemFlatStoreDataU256Jmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1596	# define IEM_MC_STORE_MEM_FLAT_U256_NO_AC(a_GCPtrMem, a_u256Value) \
1597	iemMemFlatStoreDataU256NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1598	# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1599	iemMemFlatStoreDataU256AlignedAvxJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1600	#endif
1601
1602	/* Regular stack push and pop: */
1603	#ifndef IEM_WITH_SETJMP
1604	# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1605	# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1606	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1607	# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1608
1609	# define IEM_MC_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1610	# define IEM_MC_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1611	# define IEM_MC_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1612	#else
1613	# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1614	# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1615	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1616	# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1617
1618	# define IEM_MC_POP_GREG_U16(a_iGReg) iemMemStackPopGRegU16Jmp(pVCpu, (a_iGReg))
1619	# define IEM_MC_POP_GREG_U32(a_iGReg) iemMemStackPopGRegU32Jmp(pVCpu, (a_iGReg))
1620	# define IEM_MC_POP_GREG_U64(a_iGReg) iemMemStackPopGRegU64Jmp(pVCpu, (a_iGReg))
1621	#endif
1622
1623	/* 32-bit flat stack push and pop: */
1624	#ifndef IEM_WITH_SETJMP
1625	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1626	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1627	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1628
1629	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1630	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1631	#else
1632	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1633	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1634	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1635
1636	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) iemMemFlat32StackPopGRegU16Jmp(pVCpu, a_iGReg))
1637	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) iemMemFlat32StackPopGRegU32Jmp(pVCpu, a_iGReg))
1638	#endif
1639
1640	/* 64-bit flat stack push and pop: */
1641	#ifndef IEM_WITH_SETJMP
1642	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1643	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1644
1645	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1646	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1647	#else
1648	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1649	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1650
1651	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) iemMemFlat64StackPopGRegU16Jmp(pVCpu, (a_iGReg))
1652	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) iemMemFlat64StackPopGRegU64Jmp(pVCpu, (a_iGReg))
1653	#endif
1654
1655
1656	/* 8-bit */
1657
1658	/**
1659	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1660	* acccess, for atomic operations.
1661	*
1662	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1663	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1664	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1665	* @param[in] a_GCPtrMem The memory address.
1666	* @remarks Will return/long jump on errors.
1667	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1668	*/
1669	#ifndef IEM_WITH_SETJMP
1670	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1671	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1672	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1673	#else
1674	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1675	(a_pu8Mem) = iemMemMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1676	#endif
1677
1678	/**
1679	* Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1680	*
1681	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1682	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1683	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1684	* @param[in] a_GCPtrMem The memory address.
1685	* @remarks Will return/long jump on errors.
1686	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1687	*/
1688	#ifndef IEM_WITH_SETJMP
1689	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1690	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1691	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1692	#else
1693	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1694	(a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1695	#endif
1696
1697	/**
1698	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1699	*
1700	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1701	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1702	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1703	* @param[in] a_GCPtrMem The memory address.
1704	* @remarks Will return/long jump on errors.
1705	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1706	*/
1707	#ifndef IEM_WITH_SETJMP
1708	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1709	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1710	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1711	#else
1712	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1713	(a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1714	#endif
1715
1716	/**
1717	* Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1718	*
1719	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1720	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1721	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1722	* @param[in] a_GCPtrMem The memory address.
1723	* @remarks Will return/long jump on errors.
1724	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1725	*/
1726	#ifndef IEM_WITH_SETJMP
1727	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1728	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1729	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1730	#else
1731	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1732	(a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1733	#endif
1734
1735	/**
1736	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1737	* acccess, flat address variant.
1738	*
1739	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1740	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1741	* @param[in] a_GCPtrMem The memory address.
1742	* @remarks Will return/long jump on errors.
1743	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1744	*/
1745	#ifndef IEM_WITH_SETJMP
1746	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1747	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1748	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1749	#else
1750	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1751	(a_pu8Mem) = iemMemFlatMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1752	#endif
1753
1754	/**
1755	* Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1756	* address variant.
1757	*
1758	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1759	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1760	* @param[in] a_GCPtrMem The memory address.
1761	* @remarks Will return/long jump on errors.
1762	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1763	*/
1764	#ifndef IEM_WITH_SETJMP
1765	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1766	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1767	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1768	#else
1769	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1770	(a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1771	#endif
1772
1773	/**
1774	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1775	* address variant.
1776	*
1777	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1778	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1779	* @param[in] a_GCPtrMem The memory address.
1780	* @remarks Will return/long jump on errors.
1781	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1782	*/
1783	#ifndef IEM_WITH_SETJMP
1784	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1785	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1786	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1787	#else
1788	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1789	(a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1790	#endif
1791
1792	/**
1793	* Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1794	* address variant.
1795	*
1796	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1797	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1798	* @param[in] a_GCPtrMem The memory address.
1799	* @remarks Will return/long jump on errors.
1800	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1801	*/
1802	#ifndef IEM_WITH_SETJMP
1803	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1804	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1805	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1806	#else
1807	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1808	(a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1809	#endif
1810
1811
1812	/* 16-bit */
1813
1814	/**
1815	* Maps guest memory for word atomic read+write direct (or bounce) buffer acccess.
1816	*
1817	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1818	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1819	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1820	* @param[in] a_GCPtrMem The memory address.
1821	* @remarks Will return/long jump on errors.
1822	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1823	*/
1824	#ifndef IEM_WITH_SETJMP
1825	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1826	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1827	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1828	#else
1829	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1830	(a_pu16Mem) = iemMemMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1831	#endif
1832
1833	/**
1834	* Maps guest memory for word read+write direct (or bounce) buffer acccess.
1835	*
1836	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1837	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1838	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1839	* @param[in] a_GCPtrMem The memory address.
1840	* @remarks Will return/long jump on errors.
1841	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1842	*/
1843	#ifndef IEM_WITH_SETJMP
1844	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1845	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1846	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1847	#else
1848	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1849	(a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1850	#endif
1851
1852	/**
1853	* Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1854	*
1855	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1856	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1857	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1858	* @param[in] a_GCPtrMem The memory address.
1859	* @remarks Will return/long jump on errors.
1860	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1861	*/
1862	#ifndef IEM_WITH_SETJMP
1863	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1864	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1865	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1866	#else
1867	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1868	(a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1869	#endif
1870
1871	/**
1872	* Maps guest memory for word readonly direct (or bounce) buffer acccess.
1873	*
1874	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1875	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1876	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1877	* @param[in] a_GCPtrMem The memory address.
1878	* @remarks Will return/long jump on errors.
1879	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1880	*/
1881	#ifndef IEM_WITH_SETJMP
1882	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1883	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1884	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1885	#else
1886	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1887	(a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1888	#endif
1889
1890	/**
1891	* Maps guest memory for word atomic read+write direct (or bounce) buffer
1892	* acccess, flat address variant.
1893	*
1894	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1895	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1896	* @param[in] a_GCPtrMem The memory address.
1897	* @remarks Will return/long jump on errors.
1898	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1899	*/
1900	#ifndef IEM_WITH_SETJMP
1901	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1902	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1903	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1904	#else
1905	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1906	(a_pu16Mem) = iemMemFlatMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1907	#endif
1908
1909	/**
1910	* Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1911	* address variant.
1912	*
1913	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1914	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1915	* @param[in] a_GCPtrMem The memory address.
1916	* @remarks Will return/long jump on errors.
1917	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1918	*/
1919	#ifndef IEM_WITH_SETJMP
1920	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1921	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1922	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1923	#else
1924	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1925	(a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1926	#endif
1927
1928	/**
1929	* Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1930	* address variant.
1931	*
1932	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1933	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1934	* @param[in] a_GCPtrMem The memory address.
1935	* @remarks Will return/long jump on errors.
1936	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1937	*/
1938	#ifndef IEM_WITH_SETJMP
1939	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1940	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1941	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1942	#else
1943	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1944	(a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1945	#endif
1946
1947	/**
1948	* Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1949	* address variant.
1950	*
1951	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1952	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1953	* @param[in] a_GCPtrMem The memory address.
1954	* @remarks Will return/long jump on errors.
1955	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1956	*/
1957	#ifndef IEM_WITH_SETJMP
1958	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1959	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1960	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1961	#else
1962	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1963	(a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1964	#endif
1965
1966	/** int16_t alias. */
1967	#ifndef IEM_WITH_SETJMP
1968	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1969	IEM_MC_MEM_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
1970	#else
1971	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1972	(a_pi16Mem) = (int16_t *)iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1973	#endif
1974
1975	/** Flat int16_t alias. */
1976	#ifndef IEM_WITH_SETJMP
1977	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1978	IEM_MC_MEM_FLAT_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem)
1979	#else
1980	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1981	(a_pi16Mem) = (int16_t *)iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1982	#endif
1983
1984
1985	/* 32-bit */
1986
1987	/**
1988	* Maps guest memory for dword atomic read+write direct (or bounce) buffer acccess.
1989	*
1990	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1991	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1992	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1993	* @param[in] a_GCPtrMem The memory address.
1994	* @remarks Will return/long jump on errors.
1995	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1996	*/
1997	#ifndef IEM_WITH_SETJMP
1998	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1999	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2000	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
2001	#else
2002	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2003	(a_pu32Mem) = iemMemMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2004	#endif
2005
2006	/**
2007	* Maps guest memory for dword read+write direct (or bounce) buffer acccess.
2008	*
2009	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2010	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2011	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2012	* @param[in] a_GCPtrMem The memory address.
2013	* @remarks Will return/long jump on errors.
2014	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2015	*/
2016	#ifndef IEM_WITH_SETJMP
2017	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2018	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2019	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2020	#else
2021	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2022	(a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2023	#endif
2024
2025	/**
2026	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
2027	*
2028	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2029	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2030	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2031	* @param[in] a_GCPtrMem The memory address.
2032	* @remarks Will return/long jump on errors.
2033	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2034	*/
2035	#ifndef IEM_WITH_SETJMP
2036	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2037	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2038	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2039	#else
2040	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2041	(a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2042	#endif
2043
2044	/**
2045	* Maps guest memory for dword readonly direct (or bounce) buffer acccess.
2046	*
2047	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2048	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2049	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2050	* @param[in] a_GCPtrMem The memory address.
2051	* @remarks Will return/long jump on errors.
2052	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2053	*/
2054	#ifndef IEM_WITH_SETJMP
2055	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2056	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2057	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2058	#else
2059	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2060	(a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2061	#endif
2062
2063	/**
2064	* Maps guest memory for dword atomic read+write direct (or bounce) buffer
2065	* acccess, flat address variant.
2066	*
2067	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2068	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2069	* @param[in] a_GCPtrMem The memory address.
2070	* @remarks Will return/long jump on errors.
2071	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2072	*/
2073	#ifndef IEM_WITH_SETJMP
2074	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2075	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2076	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
2077	#else
2078	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2079	(a_pu32Mem) = iemMemFlatMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2080	#endif
2081
2082	/**
2083	* Maps guest memory for dword read+write direct (or bounce) buffer acccess,
2084	* flat address variant.
2085	*
2086	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2087	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2088	* @param[in] a_GCPtrMem The memory address.
2089	* @remarks Will return/long jump on errors.
2090	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2091	*/
2092	#ifndef IEM_WITH_SETJMP
2093	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2094	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2095	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2096	#else
2097	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2098	(a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2099	#endif
2100
2101	/**
2102	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
2103	* address variant.
2104	*
2105	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2106	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2107	* @param[in] a_GCPtrMem The memory address.
2108	* @remarks Will return/long jump on errors.
2109	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2110	*/
2111	#ifndef IEM_WITH_SETJMP
2112	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2113	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2114	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2115	#else
2116	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2117	(a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2118	#endif
2119
2120	/**
2121	* Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
2122	* address variant.
2123	*
2124	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2125	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2126	* @param[in] a_GCPtrMem The memory address.
2127	* @remarks Will return/long jump on errors.
2128	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2129	*/
2130	#ifndef IEM_WITH_SETJMP
2131	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2132	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2133	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2134	#else
2135	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2136	(a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2137	#endif
2138
2139	/** int32_t alias. */
2140	#ifndef IEM_WITH_SETJMP
2141	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2142	IEM_MC_MEM_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2143	#else
2144	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2145	(a_pi32Mem) = (int32_t *)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2146	#endif
2147
2148	/** Flat int32_t alias. */
2149	#ifndef IEM_WITH_SETJMP
2150	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2151	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem)
2152	#else
2153	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2154	(a_pi32Mem) = (int32_t *)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2155	#endif
2156
2157	/** RTFLOAT32U alias. */
2158	#ifndef IEM_WITH_SETJMP
2159	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2160	IEM_MC_MEM_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2161	#else
2162	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2163	(a_pr32Mem) = (PRTFLOAT32U)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2164	#endif
2165
2166	/** Flat RTFLOAT32U alias. */
2167	#ifndef IEM_WITH_SETJMP
2168	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2169	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem)
2170	#else
2171	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2172	(a_pr32Mem) = (PRTFLOAT32U)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2173	#endif
2174
2175
2176	/* 64-bit */
2177
2178	/**
2179	* Maps guest memory for qword atomic read+write direct (or bounce) buffer acccess.
2180	*
2181	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2182	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2183	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2184	* @param[in] a_GCPtrMem The memory address.
2185	* @remarks Will return/long jump on errors.
2186	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2187	*/
2188	#ifndef IEM_WITH_SETJMP
2189	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2190	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2191	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2192	#else
2193	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2194	(a_pu64Mem) = iemMemMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2195	#endif
2196
2197	/**
2198	* Maps guest memory for qword read+write direct (or bounce) buffer acccess.
2199	*
2200	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2201	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2202	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2203	* @param[in] a_GCPtrMem The memory address.
2204	* @remarks Will return/long jump on errors.
2205	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2206	*/
2207	#ifndef IEM_WITH_SETJMP
2208	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2209	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2210	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2211	#else
2212	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2213	(a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2214	#endif
2215
2216	/**
2217	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
2218	*
2219	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2220	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2221	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2222	* @param[in] a_GCPtrMem The memory address.
2223	* @remarks Will return/long jump on errors.
2224	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2225	*/
2226	#ifndef IEM_WITH_SETJMP
2227	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2228	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2229	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2230	#else
2231	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2232	(a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2233	#endif
2234
2235	/**
2236	* Maps guest memory for qword readonly direct (or bounce) buffer acccess.
2237	*
2238	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2239	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2240	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2241	* @param[in] a_GCPtrMem The memory address.
2242	* @remarks Will return/long jump on errors.
2243	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2244	*/
2245	#ifndef IEM_WITH_SETJMP
2246	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2247	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2248	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2249	#else
2250	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2251	(a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2252	#endif
2253
2254	/**
2255	* Maps guest memory for qword atomic read+write direct (or bounce) buffer
2256	* acccess, flat address variant.
2257	*
2258	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2259	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2260	* @param[in] a_GCPtrMem The memory address.
2261	* @remarks Will return/long jump on errors.
2262	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2263	*/
2264	#ifndef IEM_WITH_SETJMP
2265	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2266	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2267	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2268	#else
2269	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2270	(a_pu64Mem) = iemMemFlatMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2271	#endif
2272
2273	/**
2274	* Maps guest memory for qword read+write direct (or bounce) buffer acccess,
2275	* flat address variant.
2276	*
2277	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2278	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2279	* @param[in] a_GCPtrMem The memory address.
2280	* @remarks Will return/long jump on errors.
2281	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2282	*/
2283	#ifndef IEM_WITH_SETJMP
2284	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2285	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2286	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2287	#else
2288	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2289	(a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2290	#endif
2291
2292	/**
2293	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
2294	* address variant.
2295	*
2296	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2297	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2298	* @param[in] a_GCPtrMem The memory address.
2299	* @remarks Will return/long jump on errors.
2300	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2301	*/
2302	#ifndef IEM_WITH_SETJMP
2303	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2304	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2305	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2306	#else
2307	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2308	(a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2309	#endif
2310
2311	/**
2312	* Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
2313	* address variant.
2314	*
2315	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2316	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2317	* @param[in] a_GCPtrMem The memory address.
2318	* @remarks Will return/long jump on errors.
2319	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2320	*/
2321	#ifndef IEM_WITH_SETJMP
2322	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2323	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2324	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2325	#else
2326	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2327	(a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2328	#endif
2329
2330	/** int64_t alias. */
2331	#ifndef IEM_WITH_SETJMP
2332	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2333	IEM_MC_MEM_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2334	#else
2335	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2336	(a_pi64Mem) = (int64_t *)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2337	#endif
2338
2339	/** Flat int64_t alias. */
2340	#ifndef IEM_WITH_SETJMP
2341	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2342	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem)
2343	#else
2344	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2345	(a_pi64Mem) = (int64_t *)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2346	#endif
2347
2348	/** RTFLOAT64U alias. */
2349	#ifndef IEM_WITH_SETJMP
2350	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2351	IEM_MC_MEM_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2352	#else
2353	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2354	(a_pr64Mem) = (PRTFLOAT64U)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2355	#endif
2356
2357	/** Flat RTFLOAT64U alias. */
2358	#ifndef IEM_WITH_SETJMP
2359	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2360	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem)
2361	#else
2362	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2363	(a_pr64Mem) = (PRTFLOAT64U)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2364	#endif
2365
2366
2367	/* 128-bit */
2368
2369	/**
2370	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer acccess.
2371	*
2372	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2373	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2374	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2375	* @param[in] a_GCPtrMem The memory address.
2376	* @remarks Will return/long jump on errors.
2377	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2378	*/
2379	#ifndef IEM_WITH_SETJMP
2380	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2381	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2382	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1))
2383	#else
2384	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2385	(a_pu128Mem) = iemMemMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2386	#endif
2387
2388	/**
2389	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess.
2390	*
2391	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2392	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2393	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2394	* @param[in] a_GCPtrMem The memory address.
2395	* @remarks Will return/long jump on errors.
2396	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2397	*/
2398	#ifndef IEM_WITH_SETJMP
2399	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2400	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2401	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1))
2402	#else
2403	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2404	(a_pu128Mem) = iemMemMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2405	#endif
2406
2407	/**
2408	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess.
2409	*
2410	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2411	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2412	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2413	* @param[in] a_GCPtrMem The memory address.
2414	* @remarks Will return/long jump on errors.
2415	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2416	*/
2417	#ifndef IEM_WITH_SETJMP
2418	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2419	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2420	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2421	#else
2422	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2423	(a_pu128Mem) = iemMemMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2424	#endif
2425
2426	/**
2427	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess.
2428	*
2429	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2430	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2431	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2432	* @param[in] a_GCPtrMem The memory address.
2433	* @remarks Will return/long jump on errors.
2434	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2435	*/
2436	#ifndef IEM_WITH_SETJMP
2437	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2438	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2439	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2440	#else
2441	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2442	(a_pu128Mem) = iemMemMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2443	#endif
2444
2445	/**
2446	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer
2447	* access, flat address variant.
2448	*
2449	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2450	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2451	* @param[in] a_GCPtrMem The memory address.
2452	* @remarks Will return/long jump on errors.
2453	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2454	*/
2455	#ifndef IEM_WITH_SETJMP
2456	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2457	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2458	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128) - 1))
2459	#else
2460	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2461	(a_pu128Mem) = iemMemFlatMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2462	#endif
2463
2464	/**
2465	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess,
2466	* flat address variant.
2467	*
2468	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2469	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2470	* @param[in] a_GCPtrMem The memory address.
2471	* @remarks Will return/long jump on errors.
2472	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2473	*/
2474	#ifndef IEM_WITH_SETJMP
2475	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2476	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2477	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128) - 1))
2478	#else
2479	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2480	(a_pu128Mem) = iemMemFlatMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2481	#endif
2482
2483	/**
2484	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess,
2485	* flat address variant.
2486	*
2487	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2488	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2489	* @param[in] a_GCPtrMem The memory address.
2490	* @remarks Will return/long jump on errors.
2491	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2492	*/
2493	#ifndef IEM_WITH_SETJMP
2494	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2495	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2496	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2497	#else
2498	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2499	(a_pu128Mem) = iemMemFlatMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2500	#endif
2501
2502	/**
2503	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess, flat
2504	* address variant.
2505	*
2506	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2507	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2508	* @param[in] a_GCPtrMem The memory address.
2509	* @remarks Will return/long jump on errors.
2510	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2511	*/
2512	#ifndef IEM_WITH_SETJMP
2513	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2514	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2515	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2516	#else
2517	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2518	(a_pu128Mem) = iemMemFlatMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2519	#endif
2520
2521
2522	/* misc */
2523
2524	/**
2525	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2526	*
2527	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2528	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2529	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2530	* @param[in] a_GCPtrMem The memory address.
2531	* @remarks Will return/long jump on errors.
2532	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2533	*/
2534	#ifndef IEM_WITH_SETJMP
2535	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2536	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2537	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2538	#else
2539	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2540	(a_pr80Mem) = iemMemMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2541	#endif
2542
2543	/**
2544	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2545	*
2546	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2547	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2548	* @param[in] a_GCPtrMem The memory address.
2549	* @remarks Will return/long jump on errors.
2550	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2551	*/
2552	#ifndef IEM_WITH_SETJMP
2553	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2554	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2555	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2556	#else
2557	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2558	(a_pr80Mem) = iemMemFlatMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2559	#endif
2560
2561
2562	/**
2563	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2564	*
2565	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2566	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2567	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2568	* @param[in] a_GCPtrMem The memory address.
2569	* @remarks Will return/long jump on errors.
2570	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2571	*/
2572	#ifndef IEM_WITH_SETJMP
2573	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2574	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2575	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2576	#else
2577	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2578	(a_pd80Mem) = iemMemMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2579	#endif
2580
2581	/**
2582	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2583	*
2584	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2585	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2586	* @param[in] a_GCPtrMem The memory address.
2587	* @remarks Will return/long jump on errors.
2588	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2589	*/
2590	#ifndef IEM_WITH_SETJMP
2591	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2592	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2593	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2594	#else
2595	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2596	(a_pd80Mem) = iemMemFlatMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2597	#endif
2598
2599
2600
2601	/* commit + unmap */
2602
2603	/** Commits the memory and unmaps guest memory previously mapped RW.
2604	* @remarks May return.
2605	* @note Implictly frees the a_bMapInfo variable.
2606	*/
2607	#ifndef IEM_WITH_SETJMP
2608	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2609	#else
2610	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2611	#endif
2612
2613	/** Commits the memory and unmaps guest memory previously mapped ATOMIC.
2614	* @remarks May return.
2615	* @note Implictly frees the a_bMapInfo variable.
2616	*/
2617	#ifndef IEM_WITH_SETJMP
2618	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2619	#else
2620	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2621	#endif
2622
2623	/** Commits the memory and unmaps guest memory previously mapped W.
2624	* @remarks May return.
2625	* @note Implictly frees the a_bMapInfo variable.
2626	*/
2627	#ifndef IEM_WITH_SETJMP
2628	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2629	#else
2630	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) iemMemCommitAndUnmapWoJmp(pVCpu, (a_bMapInfo))
2631	#endif
2632
2633	/** Commits the memory and unmaps guest memory previously mapped R.
2634	* @remarks May return.
2635	* @note Implictly frees the a_bMapInfo variable.
2636	*/
2637	#ifndef IEM_WITH_SETJMP
2638	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2639	#else
2640	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) iemMemCommitAndUnmapRoJmp(pVCpu, (a_bMapInfo))
2641	#endif
2642
2643
2644	/** Commits the memory and unmaps the guest memory unless the FPU status word
2645	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
2646	* that would cause FLD not to store.
2647	*
2648	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
2649	* store, while \#P will not.
2650	*
2651	* @remarks May in theory return - for now.
2652	* @note Implictly frees both the a_bMapInfo and a_u16FSW variables.
2653	*/
2654	#ifndef IEM_WITH_SETJMP
2655	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2656	if ( !(a_u16FSW & X86_FSW_ES) \
2657	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2658	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2659	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo)); \
2660	else \
2661	iemMemRollbackAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W); \
2662	} while (0)
2663	#else
2664	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2665	if ( !(a_u16FSW & X86_FSW_ES) \
2666	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2667	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2668	iemMemCommitAndUnmapWoJmp(pVCpu, a_bMapInfo); \
2669	else \
2670	iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo); \
2671	} while (0)
2672	#endif
2673
2674	/** Rolls back (conceptually only, assumes no writes) and unmaps the guest memory.
2675	* @note Implictly frees the a_bMapInfo variable. */
2676	#ifndef IEM_WITH_SETJMP
2677	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmap(pVCpu, a_bMapInfo)
2678	#else
2679	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo)
2680	#endif
2681
2682
2683
2684	/** Calculate efficient address from R/M. */
2685	#ifndef IEM_WITH_SETJMP
2686	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2687	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
2688	#else
2689	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2690	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
2691	#endif
2692
2693
2694	/** The @a a_fSupportedHosts mask are ORed together RT_ARCH_VAL_XXX values. */
2695	#define IEM_MC_NATIVE_IF(a_fSupportedHosts) if (false) {
2696	#define IEM_MC_NATIVE_ELSE() } else {
2697	#define IEM_MC_NATIVE_ENDIF() } ((void)0)
2698
2699	#define IEM_MC_NATIVE_EMIT_0(a_fnEmitter)
2700	#define IEM_MC_NATIVE_EMIT_1(a_fnEmitter, a0) (void)(a0)
2701	#define IEM_MC_NATIVE_EMIT_2(a_fnEmitter, a0, a1) (void)(a0), (void)(a1)
2702	#define IEM_MC_NATIVE_EMIT_2_EX(a_fnEmitter, a0, a1) (void)(a0), (void)(a1)
2703	#define IEM_MC_NATIVE_EMIT_3(a_fnEmitter, a0, a1, a2) (void)(a0), (void)(a1), (void)(a2)
2704	#define IEM_MC_NATIVE_EMIT_4(a_fnEmitter, a0, a1, a2, a3) (void)(a0), (void)(a1), (void)(a2), (void)(a3)
2705	#define IEM_MC_NATIVE_EMIT_5(a_fnEmitter, a0, a1, a2, a3, a4) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4)
2706	#define IEM_MC_NATIVE_EMIT_6(a_fnEmitter, a0, a1, a2, a3, a4, a5) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5)
2707	#define IEM_MC_NATIVE_EMIT_7(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6)
2708	#define IEM_MC_NATIVE_EMIT_8(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6, a7) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6), (void)(a7)
2709
2710	/** This can be used to direct the register allocator when dealing with
2711	* x86/AMD64 instructions (like SHL reg,CL) that takes fixed registers. */
2712	#define IEM_MC_NATIVE_SET_AMD64_HOST_REG_FOR_LOCAL(a_VarNm, a_idxHostReg) ((void)0)
2713
2714
2715	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
2716	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
2717	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
2718	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
2719	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
2720	#define IEM_MC_CALL_AIMPL_3(a_rcType, a_rc, a_pfn, a0, a1, a2) a_rcType const a_rc = (a_pfn)((a0), (a1), (a2))
2721	#define IEM_MC_CALL_AIMPL_4(a_rcType, a_rc, a_pfn, a0, a1, a2, a3) a_rcType const a_rc = (a_pfn)((a0), (a1), (a2), (a3))
2722
2723
2724	/** @def IEM_MC_CALL_CIMPL_HLP_RET
2725	* Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
2726	*/
2727	#ifdef VBOX_STRICT
2728	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
2729	do { \
2730	uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
2731	uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
2732	uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
2733	uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
2734	uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
2735	VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
2736	if (rcStrictHlp == VINF_SUCCESS) \
2737	{ \
2738	uint64_t const fRipMask = (pVCpu->iem.s.fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT ? UINT64_MAX : UINT32_MAX; \
2739	AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
2740	\|\| ( ((uRipBefore + cbInstr) & fRipMask) == pVCpu->cpum.GstCtx.rip \
2741	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
2742	\|\| ( ((a_fFlags) & IEM_CIMPL_F_REP) \
2743	&& uRipBefore == pVCpu->cpum.GstCtx.rip \
2744	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
2745	("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
2746	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, (uRipBefore + cbInstr) & fRipMask)); \
2747	if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
2748	{ /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
2749	else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
2750	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)) \
2751	== (fEflBefore & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)), \
2752	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2753	else \
2754	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2755	== (fEflBefore & ~(X86_EFL_RF)), \
2756	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2757	if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2758	{ \
2759	uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2760	AssertMsg( fExecBefore == fExecRecalc \
2761	/* in case ES, DS or SS was external initially (happens alot with HM): */ \
2762	\|\| ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2763	&& (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2764	("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2765	} \
2766	} \
2767	return rcStrictHlp; \
2768	} while (0)
2769	#else
2770	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2771	#endif
2772
2773	/**
2774	* Defers the rest of the instruction emulation to a C implementation routine
2775	* and returns, only taking the standard parameters.
2776	*
2777	* @param a_fFlags IEM_CIMPL_F_XXX.
2778	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2779	* in the native recompiler.
2780	* @param a_pfnCImpl The pointer to the C routine.
2781	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2782	*/
2783	#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2784	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2785
2786	/**
2787	* Defers the rest of instruction emulation to a C implementation routine and
2788	* returns, taking one argument in addition to the standard ones.
2789	*
2790	* @param a_fFlags IEM_CIMPL_F_XXX.
2791	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2792	* in the native recompiler.
2793	* @param a_pfnCImpl The pointer to the C routine.
2794	* @param a0 The argument.
2795	*/
2796	#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2797	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2798
2799	/**
2800	* Defers the rest of the instruction emulation to a C implementation routine
2801	* and returns, taking two arguments in addition to the standard ones.
2802	*
2803	* @param a_fFlags IEM_CIMPL_F_XXX.
2804	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2805	* in the native recompiler.
2806	* @param a_pfnCImpl The pointer to the C routine.
2807	* @param a0 The first extra argument.
2808	* @param a1 The second extra argument.
2809	*/
2810	#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2811	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2812
2813	/**
2814	* Defers the rest of the instruction emulation to a C implementation routine
2815	* and returns, taking three arguments in addition to the standard ones.
2816	*
2817	* @param a_fFlags IEM_CIMPL_F_XXX.
2818	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2819	* in the native recompiler.
2820	* @param a_pfnCImpl The pointer to the C routine.
2821	* @param a0 The first extra argument.
2822	* @param a1 The second extra argument.
2823	* @param a2 The third extra argument.
2824	*/
2825	#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2826	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2827
2828	/**
2829	* Defers the rest of the instruction emulation to a C implementation routine
2830	* and returns, taking four arguments in addition to the standard ones.
2831	*
2832	* @param a_fFlags IEM_CIMPL_F_XXX.
2833	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2834	* in the native recompiler.
2835	* @param a_pfnCImpl The pointer to the C routine.
2836	* @param a0 The first extra argument.
2837	* @param a1 The second extra argument.
2838	* @param a2 The third extra argument.
2839	* @param a3 The fourth extra argument.
2840	*/
2841	#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3) \
2842	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2843
2844	/**
2845	* Defers the rest of the instruction emulation to a C implementation routine
2846	* and returns, taking five arguments in addition to the standard ones.
2847	*
2848	* @param a_fFlags IEM_CIMPL_F_XXX.
2849	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2850	* in the native recompiler.
2851	* @param a_pfnCImpl The pointer to the C routine.
2852	* @param a0 The first extra argument.
2853	* @param a1 The second extra argument.
2854	* @param a2 The third extra argument.
2855	* @param a3 The fourth extra argument.
2856	* @param a4 The fifth extra argument.
2857	*/
2858	#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3, a4) \
2859	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2860
2861	/**
2862	* Defers the entire instruction emulation to a C implementation routine and
2863	* returns, only taking the standard parameters.
2864	*
2865	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2866	*
2867	* @param a_fFlags IEM_CIMPL_F_XXX.
2868	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2869	* in the native recompiler.
2870	* @param a_pfnCImpl The pointer to the C routine.
2871	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2872	*/
2873	#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2874	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2875
2876	/**
2877	* Defers the entire instruction emulation to a C implementation routine and
2878	* returns, taking one argument in addition to the standard ones.
2879	*
2880	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2881	*
2882	* @param a_fFlags IEM_CIMPL_F_XXX.
2883	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2884	* in the native recompiler.
2885	* @param a_pfnCImpl The pointer to the C routine.
2886	* @param a0 The argument.
2887	*/
2888	#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2889	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2890
2891	/**
2892	* Defers the entire instruction emulation to a C implementation routine and
2893	* returns, taking two arguments in addition to the standard ones.
2894	*
2895	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2896	*
2897	* @param a_fFlags IEM_CIMPL_F_XXX.
2898	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2899	* in the native recompiler.
2900	* @param a_pfnCImpl The pointer to the C routine.
2901	* @param a0 The first extra argument.
2902	* @param a1 The second extra argument.
2903	*/
2904	#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2905	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2906
2907	/**
2908	* Defers the entire instruction emulation to a C implementation routine and
2909	* returns, taking three arguments in addition to the standard ones.
2910	*
2911	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2912	*
2913	* @param a_fFlags IEM_CIMPL_F_XXX.
2914	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2915	* in the native recompiler.
2916	* @param a_pfnCImpl The pointer to the C routine.
2917	* @param a0 The first extra argument.
2918	* @param a1 The second extra argument.
2919	* @param a2 The third extra argument.
2920	*/
2921	#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2922	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2923
2924
2925	/**
2926	* Calls a FPU assembly implementation taking one visible argument.
2927	*
2928	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2929	* @param a0 The first extra argument.
2930	*/
2931	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2932	do { \
2933	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2934	} while (0)
2935
2936	/**
2937	* Calls a FPU assembly implementation taking two visible arguments.
2938	*
2939	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2940	* @param a0 The first extra argument.
2941	* @param a1 The second extra argument.
2942	*/
2943	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2944	do { \
2945	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2946	} while (0)
2947
2948	/**
2949	* Calls a FPU assembly implementation taking three visible arguments.
2950	*
2951	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2952	* @param a0 The first extra argument.
2953	* @param a1 The second extra argument.
2954	* @param a2 The third extra argument.
2955	*/
2956	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2957	do { \
2958	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2959	} while (0)
2960
2961	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2962	do { \
2963	(a_FpuData).FSW = (a_FSW); \
2964	(a_FpuData).r80Result = *(a_pr80Value); \
2965	} while (0)
2966
2967	/** Pushes FPU result onto the stack. */
2968	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2969	iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2970	/** Pushes FPU result onto the stack and sets the FPUDP. */
2971	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2972	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2973
2974	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2975	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2976	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2977
2978	/** Stores FPU result in a stack register. */
2979	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2980	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2981	/** Stores FPU result in a stack register and pops the stack. */
2982	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
2983	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2984	/** Stores FPU result in a stack register and sets the FPUDP. */
2985	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2986	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2987	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
2988	* stack. */
2989	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2990	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2991
2992	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
2993	#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
2994	iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
2995	/** Free a stack register (for FFREE and FFREEP). */
2996	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
2997	iemFpuStackFree(pVCpu, a_iStReg)
2998	/** Increment the FPU stack pointer. */
2999	#define IEM_MC_FPU_STACK_INC_TOP() \
3000	iemFpuStackIncTop(pVCpu)
3001	/** Decrement the FPU stack pointer. */
3002	#define IEM_MC_FPU_STACK_DEC_TOP() \
3003	iemFpuStackDecTop(pVCpu)
3004
3005	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
3006	#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
3007	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
3008	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
3009	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
3010	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
3011	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
3012	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3013	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3014	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
3015	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
3016	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
3017	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
3018	* stack. */
3019	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3020	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3021	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
3022	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
3023	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
3024
3025	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
3026	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
3027	iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
3028	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
3029	* stack. */
3030	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
3031	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
3032	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
3033	* FPUDS. */
3034	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3035	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3036	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
3037	* FPUDS. Pops stack. */
3038	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3039	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3040	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
3041	* stack twice. */
3042	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
3043	iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
3044	/** Raises a FPU stack underflow exception for an instruction pushing a result
3045	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
3046	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
3047	iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
3048	/** Raises a FPU stack underflow exception for an instruction pushing a result
3049	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
3050	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
3051	iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
3052
3053	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
3054	* FPUIP, FPUCS and FOP. */
3055	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
3056	iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
3057	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
3058	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
3059	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3060	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3061	/** Prepares for using the FPU state.
3062	* Ensures that we can use the host FPU in the current context (RC+R0.
3063	* Ensures the guest FPU state in the CPUMCTX is up to date. */
3064	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
3065	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
3066	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
3067	/** Actualizes the guest FPU state so it can be accessed and modified. */
3068	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
3069
3070	/** Prepares for using the SSE state.
3071	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
3072	* Ensures the guest SSE state in the CPUMCTX is up to date. */
3073	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
3074	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3075	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
3076	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
3077	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
3078
3079	/** Prepares for using the AVX state.
3080	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
3081	* Ensures the guest AVX state in the CPUMCTX is up to date.
3082	* @note This will include the AVX512 state too when support for it is added
3083	* due to the zero extending feature of VEX instruction. */
3084	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
3085	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3086	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
3087	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
3088	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
3089
3090	/**
3091	* Calls a MMX assembly implementation taking two visible arguments.
3092	*
3093	* @param a_pfnAImpl Pointer to the assembly MMX routine.
3094	* @param a0 The first extra argument.
3095	* @param a1 The second extra argument.
3096	*/
3097	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
3098	do { \
3099	IEM_MC_PREPARE_FPU_USAGE(); \
3100	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
3101	} while (0)
3102
3103	/**
3104	* Calls a MMX assembly implementation taking three visible arguments.
3105	*
3106	* @param a_pfnAImpl Pointer to the assembly MMX routine.
3107	* @param a0 The first extra argument.
3108	* @param a1 The second extra argument.
3109	* @param a2 The third extra argument.
3110	*/
3111	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3112	do { \
3113	IEM_MC_PREPARE_FPU_USAGE(); \
3114	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
3115	} while (0)
3116
3117
3118	/**
3119	* Calls a SSE assembly implementation taking two visible arguments.
3120	*
3121	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3122	* @param a0 The first extra argument.
3123	* @param a1 The second extra argument.
3124	*
3125	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3126	* which is unmasked in the guest's MXCSR.
3127	*/
3128	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
3129	do { \
3130	IEM_MC_PREPARE_SSE_USAGE(); \
3131	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3132	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3133	(a0), (a1)); \
3134	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3135	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3136	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3137	{ /* probable */ } \
3138	else \
3139	{ \
3140	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3141	return iemRaiseSimdFpException(pVCpu); \
3142	return iemRaiseUndefinedOpcode(pVCpu); \
3143	} \
3144	} while (0)
3145
3146	/**
3147	* Calls a SSE assembly implementation taking three visible arguments.
3148	*
3149	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3150	* @param a0 The first extra argument.
3151	* @param a1 The second extra argument.
3152	* @param a2 The third extra argument.
3153	*
3154	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3155	* which is unmasked in the guest's MXCSR.
3156	*/
3157	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3158	do { \
3159	IEM_MC_PREPARE_SSE_USAGE(); \
3160	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3161	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3162	(a0), (a1), (a2)); \
3163	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3164	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3165	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3166	{ /* probable */ } \
3167	else \
3168	{ \
3169	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3170	return iemRaiseSimdFpException(pVCpu); \
3171	return iemRaiseUndefinedOpcode(pVCpu); \
3172	} \
3173	} while (0)
3174
3175
3176	/**
3177	* Calls a AVX assembly implementation taking two visible arguments.
3178	*
3179	* There is one implicit zero'th argument, a pointer to the extended state.
3180	*
3181	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3182	* @param a0 The first extra argument.
3183	* @param a1 The second extra argument.
3184	*
3185	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3186	* which is unmasked in the guest's MXCSR.
3187	*/
3188	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a0, a1) \
3189	do { \
3190	IEM_MC_PREPARE_AVX_USAGE(); \
3191	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3192	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3193	(a0), (a1)); \
3194	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3195	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3196	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3197	{ /* probable */ } \
3198	else \
3199	{ \
3200	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3201	return iemRaiseSimdFpException(pVCpu); \
3202	return iemRaiseUndefinedOpcode(pVCpu); \
3203	} \
3204	} while (0)
3205
3206	/**
3207	* Calls a AVX assembly implementation taking three visible arguments.
3208	*
3209	* There is one implicit zero'th argument, a pointer to the extended state.
3210	*
3211	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3212	* @param a0 The first extra argument.
3213	* @param a1 The second extra argument.
3214	* @param a2 The third extra argument.
3215	*
3216	* @note This throws an \#XF/\#UD exception if the helper indicates an exception
3217	* which is unmasked in the guest's MXCSR.
3218	*/
3219	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3220	do { \
3221	IEM_MC_PREPARE_AVX_USAGE(); \
3222	const uint32_t fMxcsrOld = pVCpu->cpum.GstCtx.XState.x87.MXCSR; \
3223	const uint32_t fMxcsrNew = a_pfnAImpl(fMxcsrOld & ~X86_MXCSR_XCPT_FLAGS, \
3224	(a0), (a1), (a2)); \
3225	pVCpu->cpum.GstCtx.XState.x87.MXCSR \|= fMxcsrNew; \
3226	if (RT_LIKELY(( ~((fMxcsrOld & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
3227	& (fMxcsrNew & X86_MXCSR_XCPT_FLAGS)) == 0)) \
3228	{ /* probable */ } \
3229	else \
3230	{ \
3231	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
3232	return iemRaiseSimdFpException(pVCpu); \
3233	return iemRaiseUndefinedOpcode(pVCpu); \
3234	} \
3235	} while (0)
3236
3237	/** @note Not for IOPL or IF testing. */
3238	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
3239	/** @note Not for IOPL or IF testing. */
3240	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
3241	/** @note Not for IOPL or IF testing. */
3242	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
3243	/** @note Not for IOPL or IF testing. */
3244	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
3245	/** @note Not for IOPL or IF testing. */
3246	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
3247	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3248	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3249	/** @note Not for IOPL or IF testing. */
3250	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
3251	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3252	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3253	/** @note Not for IOPL or IF testing. */
3254	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
3255	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3256	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3257	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3258	/** @note Not for IOPL or IF testing. */
3259	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
3260	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3261	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3262	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3263	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
3264	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
3265	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
3266	#define IEM_MC_IF_CX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.cx != 1) {
3267	#define IEM_MC_IF_ECX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.ecx != 1) {
3268	#define IEM_MC_IF_RCX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.rcx != 1) {
3269	/** @note Not for IOPL or IF testing. */
3270	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3271	if ( pVCpu->cpum.GstCtx.cx != 1 \
3272	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3273	/** @note Not for IOPL or IF testing. */
3274	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3275	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3276	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3277	/** @note Not for IOPL or IF testing. */
3278	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3279	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3280	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3281	/** @note Not for IOPL or IF testing. */
3282	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3283	if ( pVCpu->cpum.GstCtx.cx != 1 \
3284	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3285	/** @note Not for IOPL or IF testing. */
3286	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3287	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3288	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3289	/** @note Not for IOPL or IF testing. */
3290	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3291	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3292	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3293	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
3294	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
3295
3296	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
3297	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
3298	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
3299	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
3300	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
3301	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
3302	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
3303	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
3304	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
3305	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
3306	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
3307	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
3308	#define IEM_MC_IF_FCW_IM() \
3309	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
3310
3311	#define IEM_MC_ELSE() } else {
3312	#define IEM_MC_ENDIF() } do {} while (0)
3313
3314
3315	/** Recompiler debugging: Flush guest register shadow copies. */
3316	#define IEM_MC_HINT_FLUSH_GUEST_SHADOW(g_fGstShwFlush) ((void)0)
3317
3318	/** Recompiler liveness info: input GPR */
3319	#define IEM_MC_LIVENESS_GREG_INPUT(a_iGReg) ((void)0)
3320	/** Recompiler liveness info: clobbered GPR */
3321	#define IEM_MC_LIVENESS_GREG_CLOBBER(a_iGReg) ((void)0)
3322	/** Recompiler liveness info: modified GPR register (i.e. input & output) */
3323	#define IEM_MC_LIVENESS_GREG_MODIFY(a_iGReg) ((void)0)
3324
3325	/** Recompiler liveness info: input MM register */
3326	#define IEM_MC_LIVENESS_MREG_INPUT(a_iMReg) ((void)0)
3327	/** Recompiler liveness info: clobbered MM register */
3328	#define IEM_MC_LIVENESS_MREG_CLOBBER(a_iMReg) ((void)0)
3329	/** Recompiler liveness info: modified MM register (i.e. input & output) */
3330	#define IEM_MC_LIVENESS_MREG_MODIFY(a_iMReg) ((void)0)
3331
3332	/** Recompiler liveness info: input SSE register */
3333	#define IEM_MC_LIVENESS_XREG_INPUT(a_iXReg) ((void)0)
3334	/** Recompiler liveness info: clobbered SSE register */
3335	#define IEM_MC_LIVENESS_XREG_CLOBBER(a_iXReg) ((void)0)
3336	/** Recompiler liveness info: modified SSE register (i.e. input & output) */
3337	#define IEM_MC_LIVENESS_XREG_MODIFY(a_iXReg) ((void)0)
3338
3339	/** Recompiler liveness info: input MXCSR */
3340	#define IEM_MC_LIVENESS_MXCSR_INPUT() ((void)0)
3341	/** Recompiler liveness info: clobbered MXCSR */
3342	#define IEM_MC_LIVENESS_MXCSR_CLOBBER() ((void)0)
3343	/** Recompiler liveness info: modified MXCSR (i.e. input & output) */
3344	#define IEM_MC_LIVENESS_MXCSR_MODIFY() ((void)0)
3345
3346	/** @todo add more as needed. */
3347
3348	/** @} */
3349
3350	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
3351

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 106695

Download in other formats: