IEMMc.h@ 100591

Last change on this file since 100591 was 100591, checked in by vboxsync, 17 months ago
VMM/IEM: Must pass the FPU opcode word to the various MCs updating FOP as IEMCPU::uFpuOpcode isn't available during recompiled code execution. bugref:10369
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 99.3 KB

Line
1	/* $Id: IEMMc.h 100591 2023-07-15 01:20:13Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29	#define VMM_INCLUDED_SRC_include_IEMMc_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @name "Microcode" macros.
36	*
37	* The idea is that we should be able to use the same code to interpret
38	* instructions as well as recompiler instructions. Thus this obfuscation.
39	*
40	* @{
41	*/
42	#define IEM_MC_BEGIN(a_cArgs, a_cLocals) {
43	#define IEM_MC_END() }
44
45	/** Internal macro. */
46	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
47	do \
48	{ \
49	VBOXSTRICTRC rcStrict2 = a_Expr; \
50	if (rcStrict2 != VINF_SUCCESS) \
51	return rcStrict2; \
52	} while (0)
53
54
55	/** Advances RIP, finishes the instruction and returns.
56	* This may include raising debug exceptions and such. */
57	#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
58	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
59	#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
60	return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
61	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
62	* @note only usable in 16-bit op size mode. */
63	#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
64	return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
65	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
66	#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
67	return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
68	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
69	#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) return iemRegRipJumpU16AndFinishClearningRF((pVCpu), (a_u16NewIP))
70	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
71	#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) return iemRegRipJumpU32AndFinishClearningRF((pVCpu), (a_u32NewIP))
72	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
73	#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) return iemRegRipJumpU64AndFinishClearningRF((pVCpu), (a_u64NewIP))
74
75	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
76	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
77	do { \
78	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)))) \
79	{ /* probable */ } \
80	else return iemRaiseDeviceNotAvailable(pVCpu); \
81	} while (0)
82	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
83	do { \
84	if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)))) \
85	{ /* probable */ } \
86	else return iemRaiseDeviceNotAvailable(pVCpu); \
87	} while (0)
88	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
89	do { \
90	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
91	{ /* probable */ } \
92	else return iemRaiseMathFault(pVCpu); \
93	} while (0)
94	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
95	do { \
96	/* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
97	be reduced to a single compare branch in the more probably code path. */ \
98	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) \
99	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
100	\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
101	== (XSAVE_C_YMM \| XSAVE_C_SSE \| X86_CR4_OSXSAVE))) \
102	{ /* probable */ } \
103	else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
104	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
105	return iemRaiseUndefinedOpcode(pVCpu); \
106	else \
107	return iemRaiseDeviceNotAvailable(pVCpu); \
108	} while (0)
109	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR4_OSXSAVE));
110	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR0_TS));
111	AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
112	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
113	do { \
114	/* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
115	single compare branch in the more probable code path. */ \
116	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
117	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
118	== X86_CR4_OSFXSR)) \
119	{ /* likely */ } \
120	else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
121	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
122	return iemRaiseUndefinedOpcode(pVCpu); \
123	else \
124	return iemRaiseDeviceNotAvailable(pVCpu); \
125	} while (0)
126	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_CR4_OSFXSR));
127	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
128	do { \
129	/* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
130	single compare branch in the more probable code path. */ \
131	if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
132	\| (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
133	{ /* probable */ } \
134	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
135	return iemRaiseUndefinedOpcode(pVCpu); \
136	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
137	return iemRaiseDeviceNotAvailable(pVCpu); \
138	else \
139	return iemRaiseMathFault(pVCpu); \
140	} while (0)
141	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_FSW_ES));
142	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
143	do { \
144	if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
145	else return iemRaiseGeneralProtectionFault0(pVCpu); \
146	} while (0)
147	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
148	do { \
149	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
150	else return iemRaiseGeneralProtectionFault0(pVCpu); \
151	} while (0)
152	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
153	do { \
154	if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) \| IEM_GET_CPU_MODE(pVCpu)) \
155	== (X86_CR4_FSGSBASE \| IEMMODE_64BIT))) \
156	{ /* probable */ } \
157	else return iemRaiseUndefinedOpcode(pVCpu); \
158	} while (0)
159	AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
160	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
161	do { \
162	if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
163	else return iemRaiseGeneralProtectionFault0(pVCpu); \
164	} while (0)
165	#define IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
166	do { \
167	if (RT_LIKELY(( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
168	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)) \
169	{ /* probable */ } \
170	else \
171	{ \
172	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
173	return iemRaiseSimdFpException(pVCpu); \
174	return iemRaiseUndefinedOpcode(pVCpu); \
175	} \
176	} while (0)
177	#define IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
178	do { \
179	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT)\
180	return iemRaiseSimdFpException(pVCpu); \
181	return iemRaiseUndefinedOpcode(pVCpu); \
182	} while (0)
183
184
185	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
186	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
187	#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
188	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
189	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
190	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
191	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
192	uint32_t a_Name; \
193	uint32_t *a_pName = &a_Name
194	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
195	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
196
197	#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
198	#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
199	#define IEM_MC_ASSIGN_U8_SX_U64(a_u64VarOrArg, a_u8CVariableOrConst) \
200	(a_u64VarOrArg) = (int8_t)(a_u8CVariableOrConst)
201	#define IEM_MC_ASSIGN_U32_SX_U64(a_u64VarOrArg, a_u32CVariableOrConst) \
202	(a_u64VarOrArg) = (int32_t)(a_u32CVariableOrConst)
203
204	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
205	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
206	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
207	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
208	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
209	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
210	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
211	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
212	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
213	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
214	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
215	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
216	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
217	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
218	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
219	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
220	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
221	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
222	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
223	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
224	} while (0)
225	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
226	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
227	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
228	} while (0)
229	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
230	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
231	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
232	} while (0)
233	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
234	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
235	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
236	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
237	} while (0)
238	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
239	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
240	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
241	} while (0)
242	/** @note Not for IOPL or IF testing or modification. */
243	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
244	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u
245	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
246	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
247
248	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
249	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
250	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
251	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
252	#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
253	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
254	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
255	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
256	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
257	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
258	#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
259	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
260	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
261	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
262	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
263	} while (0)
264	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
265	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
266	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
267	} while (0)
268	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
269	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
270
271
272	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
273	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
274	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
275	* Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
276	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
277	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
278	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
279	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
280	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
281	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
282	/** @note Not for IOPL or IF testing or modification.
283	* @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
284	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
285	#define IEM_MC_REF_MXCSR(a_pfMxcsr) (a_pfMxcsr) = &pVCpu->cpum.GstCtx.XState.x87.MXCSR
286
287	#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) += (a_u8Value)
288	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
289	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
290	do { \
291	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
292	*pu32Reg += (a_u32Value); \
293	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
294	} while (0)
295	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
296
297	#define IEM_MC_SUB_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) -= (a_u8Value)
298	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u16Value)
299	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
300	do { \
301	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
302	*pu32Reg -= (a_u32Value); \
303	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
304	} while (0)
305	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u64Value)
306	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
307
308	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
309	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
310	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
311	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
312	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
313	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
314	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
315
316	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
317	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
318	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
319	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
320
321	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
322	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
323	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
324
325	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
326	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
327	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
328
329	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
330	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
331	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
332
333	#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
334
335	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
336	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
337	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
338
339	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
340
341	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
342
343	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
344	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
345	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
346	do { \
347	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
348	*pu32Reg &= (a_u32Value); \
349	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
350	} while (0)
351	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
352
353	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
354	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
355	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
356	do { \
357	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
358	*pu32Reg \|= (a_u32Value); \
359	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
360	} while (0)
361	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
362
363	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
364	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
365	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
366
367	/** @note Not for IOPL or IF modification. */
368	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
369	/** @note Not for IOPL or IF modification. */
370	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
371	/** @note Not for IOPL or IF modification. */
372	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
373
374	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
375
376	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
377	#define IEM_MC_FPU_TO_MMX_MODE() do { \
378	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
379	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
380	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
381	} while (0)
382
383	/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
384	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
385	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
386	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
387	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
388	} while (0)
389
390	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
391	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
392	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
393	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[0]; } while (0)
394	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
395	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
396	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
397	} while (0)
398	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
399	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
400	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
401	} while (0)
402	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
403	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
404	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
405	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
406	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
407	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
408	#define IEM_MC_MODIFIED_MREG(a_iMReg) \
409	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
410	#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
411	do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
412
413	#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
414	do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
415	if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
416	if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
417	if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
418	} while (0)
419	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
420	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
421	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
422	} while (0)
423	#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
424	do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
425	(a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
426	} while (0)
427	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
428	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
429	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
430	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
431	#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
432	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
433	#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
434	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iByte)]; } while (0)
435	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
436	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
437	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
438	} while (0)
439	#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
440	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
441	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
442	} while (0)
443	#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
444	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
445	#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
446	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
447	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
448	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
449	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
450	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
451	#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
452	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iWord)] = (a_u16Value); } while (0)
453	#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
454	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iByte)] = (a_u8Value); } while (0)
455
456	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
457	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
458	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
459	} while (0)
460
461	#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
462	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
463	#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
464	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
465	#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
466	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
467	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
468	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
469	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
470	} while (0)
471	#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value) \
472	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u64Value); } while (0)
473
474	#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
475	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
476	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
477	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
478	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
479	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
480	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
481	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
482	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
483	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
484	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
485	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
486	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
487	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
488	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
489	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
490	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
491	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
492	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
493	} while (0)
494	#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
495	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
496	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
497	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
498	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
499	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
500	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
501	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
502	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
503	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
504	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
505	} while (0)
506	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
507	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
508	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
509	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
510	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
511	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
512	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
513	} while (0)
514	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
515	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
516	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
517	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
518	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
519	} while (0)
520
521	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
522	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
523	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
524	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
525	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
526	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
527	#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
528	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
529	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
530	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
531	#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
532	(a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
533	#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
534	(a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
535	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
536	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
537	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
538	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
539	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
540	} while (0)
541
542	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
543	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
544	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
545	} while (0)
546	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc) \
547	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
548	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
549	} while (0)
550	#define IEM_MC_FETCH_YREG_2ND_U64(a_u64Dst, a_iYRegSrc) \
551	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
552	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
553	} while (0)
554	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
555	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
556	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
557	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
558	} while (0)
559	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
560	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
561	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
562	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
563	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
564	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
565	} while (0)
566
567	#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
568	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
569	if ((a_iDQword) == 0) \
570	{ \
571	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
572	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
573	} \
574	else \
575	{ \
576	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
577	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
578	} \
579	} while (0)
580
581	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
582	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
583	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
584	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
585	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
586	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
587	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
588	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
589	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
590	} while (0)
591	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
592	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
593	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
594	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
595	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
596	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
597	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
598	} while (0)
599	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
600	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
601	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
602	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
603	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
604	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
605	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
606	} while (0)
607	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
608	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
609	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
610	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
611	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
612	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
613	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
614	} while (0)
615
616	#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
617	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
618	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
619	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
620	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
621	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
622	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
623	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
624	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
625	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
626	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
627	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
628	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
629	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
630	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
631	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
632	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
633	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
634	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
635	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
636	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
637	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
638	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
639	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
640	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
641	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
642	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
643	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
644	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
645	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
646	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
647	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
648	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
649	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
650	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
651	} while (0)
652	#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
653	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
654	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
655	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
656	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
657	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
658	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
659	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
660	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
661	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
662	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
663	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
664	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
665	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
666	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
667	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
668	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
669	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
670	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
671	} while (0)
672	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
673	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
674	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
675	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
676	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
677	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
678	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
679	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
680	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
681	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
682	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
683	} while (0)
684	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
685	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
686	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
687	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
688	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
689	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
690	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
691	} while (0)
692	#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
693	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
694	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
695	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
696	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
697	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
698	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
699	} while (0)
700
701	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
702	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
703	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
704	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
705	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
706	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
707	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
708	do { uintptr_t const iYRegTmp = (a_iYReg); \
709	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
710	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
711	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
712	} while (0)
713
714	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
715	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
716	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
717	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
718	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
719	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
720	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
721	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
722	} while (0)
723	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
724	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
725	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
726	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
727	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
728	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
729	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
730	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
731	} while (0)
732	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
733	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
734	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
735	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
736	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
737	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
738	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
739	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
740	} while (0)
741
742	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
743	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
744	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
745	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
746	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
747	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
748	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
749	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
750	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
751	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
752	} while (0)
753	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
754	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
755	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
756	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
757	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
758	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
759	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
760	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
761	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
762	} while (0)
763	#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
764	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
765	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
766	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
767	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
768	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
769	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
770	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
771	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
772	} while (0)
773	#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
774	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
775	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
776	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
777	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
778	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
779	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
780	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
781	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
782	} while (0)
783	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
784	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
785	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
786	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
787	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
788	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
789	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
790	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
791	} while (0)
792	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
793	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
794	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
795	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
796	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
797	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
798	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
799	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
800	} while (0)
801
802	#ifndef IEM_WITH_SETJMP
803	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
804	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
805	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
806	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
807	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
808	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
809	#else
810	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
811	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
812	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
813	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
814	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
815	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
816	#endif
817
818	#ifndef IEM_WITH_SETJMP
819	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
820	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
821	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
822	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
823	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
824	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
825	#else
826	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
827	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
828	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
829	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
830	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
831	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
832	#endif
833
834	#ifndef IEM_WITH_SETJMP
835	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
836	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
837	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
838	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
839	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
840	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
841	#else
842	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
843	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
844	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
845	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
846	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
847	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
848	#endif
849
850	#ifdef SOME_UNUSED_FUNCTION
851	# define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
852	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
853	#endif
854
855	#ifndef IEM_WITH_SETJMP
856	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
857	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
858	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
859	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
860	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
861	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
862	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
863	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
864	#else
865	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
866	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
867	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
868	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
869	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
870	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
871	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
872	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
873	#endif
874
875	#ifndef IEM_WITH_SETJMP
876	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
877	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
878	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
879	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
880	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
881	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
882	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
883	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
884	#else
885	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
886	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
887	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
888	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
889	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
890	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
891	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
892	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
893	#endif
894
895	#ifndef IEM_WITH_SETJMP
896	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
897	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
898	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
899	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
900	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
901	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
902
903	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
904	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
905	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
906	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
907	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
908	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
909	# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
910	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_XmmDst).au32[(a_iDWord)], (a_iSeg), (a_GCPtrMem)))
911	# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
912	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_XmmDst).au64[(a_iQWord)], (a_iSeg), (a_GCPtrMem)))
913	#else
914	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
915	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
916	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
917	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
918	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
919	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
920
921	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
922	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
923	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
924	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
925	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
926	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
927	# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
928	(a_XmmDst).au32[(a_iDWord)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
929	# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
930	(a_XmmDst).au64[(a_iQWord)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
931	#endif
932
933	#ifndef IEM_WITH_SETJMP
934	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
935	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
936	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
937	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
938	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
939	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
940
941	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
942	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
943	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
944	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
945	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
946	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
947	#else
948	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
949	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
950	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
951	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
952	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
953	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
954
955	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
956	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
957	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
958	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
959	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
960	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
961	#endif
962
963
964
965	#ifndef IEM_WITH_SETJMP
966	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
967	do { \
968	uint8_t u8Tmp; \
969	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
970	(a_u16Dst) = u8Tmp; \
971	} while (0)
972	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
973	do { \
974	uint8_t u8Tmp; \
975	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
976	(a_u32Dst) = u8Tmp; \
977	} while (0)
978	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
979	do { \
980	uint8_t u8Tmp; \
981	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
982	(a_u64Dst) = u8Tmp; \
983	} while (0)
984	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
985	do { \
986	uint16_t u16Tmp; \
987	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
988	(a_u32Dst) = u16Tmp; \
989	} while (0)
990	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
991	do { \
992	uint16_t u16Tmp; \
993	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
994	(a_u64Dst) = u16Tmp; \
995	} while (0)
996	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
997	do { \
998	uint32_t u32Tmp; \
999	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1000	(a_u64Dst) = u32Tmp; \
1001	} while (0)
1002	#else /* IEM_WITH_SETJMP */
1003	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1004	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1005	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1006	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1007	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1008	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1009	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1010	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1011	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1012	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1013	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1014	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1015	#endif /* IEM_WITH_SETJMP */
1016
1017	#ifndef IEM_WITH_SETJMP
1018	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1019	do { \
1020	uint8_t u8Tmp; \
1021	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1022	(a_u16Dst) = (int8_t)u8Tmp; \
1023	} while (0)
1024	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1025	do { \
1026	uint8_t u8Tmp; \
1027	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1028	(a_u32Dst) = (int8_t)u8Tmp; \
1029	} while (0)
1030	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1031	do { \
1032	uint8_t u8Tmp; \
1033	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1034	(a_u64Dst) = (int8_t)u8Tmp; \
1035	} while (0)
1036	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1037	do { \
1038	uint16_t u16Tmp; \
1039	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1040	(a_u32Dst) = (int16_t)u16Tmp; \
1041	} while (0)
1042	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1043	do { \
1044	uint16_t u16Tmp; \
1045	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1046	(a_u64Dst) = (int16_t)u16Tmp; \
1047	} while (0)
1048	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1049	do { \
1050	uint32_t u32Tmp; \
1051	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1052	(a_u64Dst) = (int32_t)u32Tmp; \
1053	} while (0)
1054	#else /* IEM_WITH_SETJMP */
1055	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1056	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1057	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1058	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1059	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1060	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1061	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1062	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1063	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1064	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1065	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1066	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1067	#endif /* IEM_WITH_SETJMP */
1068
1069	#ifndef IEM_WITH_SETJMP
1070	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1071	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1072	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1073	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1074	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1075	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1076	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1077	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1078	#else
1079	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1080	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1081	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1082	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1083	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1084	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1085	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1086	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1087	#endif
1088
1089	#ifndef IEM_WITH_SETJMP
1090	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1091	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1092	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1093	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1094	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1095	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1096	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1097	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1098	#else
1099	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1100	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1101	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1102	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1103	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1104	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1105	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1106	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1107	#endif
1108
1109	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1110	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1111	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1112	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1113	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1114	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1115	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1116	do { \
1117	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1118	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1119	} while (0)
1120	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1121	do { \
1122	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1123	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1124	} while (0)
1125
1126	#ifndef IEM_WITH_SETJMP
1127	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1128	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1129	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1130	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1131	#else
1132	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1133	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1134	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1135	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1136	#endif
1137
1138	#ifndef IEM_WITH_SETJMP
1139	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1140	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1141	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1142	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1143	#else
1144	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1145	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1146	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1147	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1148	#endif
1149
1150
1151	#define IEM_MC_PUSH_U16(a_u16Value) \
1152	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1153	#define IEM_MC_PUSH_U32(a_u32Value) \
1154	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1155	#define IEM_MC_PUSH_U32_SREG(a_u32Value) \
1156	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_u32Value)))
1157	#define IEM_MC_PUSH_U64(a_u64Value) \
1158	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1159
1160	#define IEM_MC_POP_U16(a_pu16Value) \
1161	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
1162	#define IEM_MC_POP_U32(a_pu32Value) \
1163	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
1164	#define IEM_MC_POP_U64(a_pu64Value) \
1165	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
1166
1167	#define IEM_MC_POP_EX_U16(a_pu16Value, a_) \
1168	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16Ex(pVCpu, (a_pu16Value), (a_pNewRsp)))
1169	#define IEM_MC_POP_EX_U32(a_pu32Value) \
1170	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
1171	#define IEM_MC_POP_EX_U64(a_pu64Value) \
1172	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
1173
1174	/** Maps guest memory for direct or bounce buffered access.
1175	* The purpose is to pass it to an operand implementation, thus the a_iArg.
1176	* @remarks May return.
1177	*/
1178	#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
1179	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void *)&(a_pMem), sizeof((a_pMem)), (a_iSeg), \
1180	(a_GCPtrMem), (a_fAccess), sizeof(*(a_pMem)) - 1))
1181
1182	/** Maps guest memory for direct or bounce buffered access.
1183	* The purpose is to pass it to an operand implementation, thus the a_iArg.
1184	* @remarks May return.
1185	*/
1186	#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_cbAlign, a_iArg) \
1187	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), \
1188	(a_GCPtrMem), (a_fAccess), (a_cbAlign)))
1189
1190	/** Commits the memory and unmaps the guest memory.
1191	* @remarks May return.
1192	*/
1193	#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
1194	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess)))
1195
1196	/** Commits the memory and unmaps the guest memory unless the FPU status word
1197	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
1198	* that would cause FLD not to store.
1199	*
1200	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
1201	* store, while \#P will not.
1202	*
1203	* @remarks May in theory return - for now.
1204	*/
1205	#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
1206	do { \
1207	if ( !(a_u16FSW & X86_FSW_ES) \
1208	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
1209	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
1210	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess))); \
1211	} while (0)
1212
1213	/** Calculate efficient address from R/M. */
1214	#ifndef IEM_WITH_SETJMP
1215	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
1216	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
1217	#else
1218	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
1219	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
1220	#endif
1221
1222	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
1223	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
1224	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
1225	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
1226	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
1227	#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
1228	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
1229
1230	/** @name IEM_CIMPL_F_XXX - State change clues for CIMPL calls.
1231	*
1232	* These clues are mainly for the recompiler, so that it can
1233	*
1234	* @{ */
1235	#define IEM_CIMPL_F_MODE RT_BIT_32(0) /*< Execution flags may change (IEMCPU::fExec). /
1236	#define IEM_CIMPL_F_BRANCH RT_BIT_32(1) /*< Branches (changes RIP, maybe CS). /
1237	#define IEM_CIMPL_F_RFLAGS RT_BIT_32(2) /*< May change significant portions of RFLAGS. /
1238	#define IEM_CIMPL_F_STATUS_FLAGS RT_BIT_32(3) /*< May change the status bits (X86_EFL_STATUS_BITS) in RFLAGS . /
1239	#define IEM_CIMPL_F_VMEXIT RT_BIT_32(4) /*< May trigger a VM exit. /
1240	#define IEM_CIMPL_F_FPU RT_BIT_32(5) /*< May modify FPU state. /
1241	#define IEM_CIMPL_F_REP RT_BIT_32(6) /*< REP prefixed instruction which may yield before updating PC. /
1242	#define IEM_CIMPL_F_END_TB RT_BIT_32(7)
1243	/** Convenience: Raise exception (technically unnecessary, since it shouldn't return VINF_SUCCESS). */
1244	#define IEM_CIMPL_F_XCPT (IEM_CIMPL_F_MODE \| IEM_CIMPL_F_BRANCH \| IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_VMEXIT)
1245	/** @} */
1246
1247	/** @def IEM_MC_CALL_CIMPL_HLP_RET
1248	* Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
1249	*/
1250	#ifdef VBOX_STRICT
1251	#define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
1252	do { \
1253	uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
1254	uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
1255	uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
1256	uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
1257	uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
1258	VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
1259	if (rcStrictHlp == VINF_SUCCESS) \
1260	{ \
1261	AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH) \
1262	\|\| ( uRipBefore + cbInstr == pVCpu->cpum.GstCtx.rip \
1263	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
1264	\|\| ( ((a_fFlags) & IEM_CIMPL_F_REP) \
1265	&& uRipBefore == pVCpu->cpum.GstCtx.rip \
1266	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
1267	("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
1268	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, uRipBefore + cbInstr)); \
1269	if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
1270	{ /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
1271	else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
1272	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)) \
1273	== (fEflBefore & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)), \
1274	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
1275	else \
1276	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
1277	== (fEflBefore & ~(X86_EFL_RF)), \
1278	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
1279	if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
1280	{ \
1281	uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
1282	AssertMsg(fExecBefore == fExecRecalc, \
1283	("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
1284	} \
1285	} \
1286	return rcStrictHlp; \
1287	} while (0)
1288	#else
1289	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
1290	#endif
1291
1292	/**
1293	* Defers the rest of the instruction emulation to a C implementation routine
1294	* and returns, only taking the standard parameters.
1295	*
1296	* @param a_fFlags IEM_CIMPL_F_XXX.
1297	* @param a_pfnCImpl The pointer to the C routine.
1298	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
1299	*/
1300	#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_pfnCImpl) \
1301	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
1302
1303	/**
1304	* Defers the rest of instruction emulation to a C implementation routine and
1305	* returns, taking one argument in addition to the standard ones.
1306	*
1307	* @param a_fFlags IEM_CIMPL_F_XXX.
1308	* @param a_pfnCImpl The pointer to the C routine.
1309	* @param a0 The argument.
1310	*/
1311	#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_pfnCImpl, a0) \
1312	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
1313
1314	/**
1315	* Defers the rest of the instruction emulation to a C implementation routine
1316	* and returns, taking two arguments in addition to the standard ones.
1317	*
1318	* @param a_fFlags IEM_CIMPL_F_XXX.
1319	* @param a_pfnCImpl The pointer to the C routine.
1320	* @param a0 The first extra argument.
1321	* @param a1 The second extra argument.
1322	*/
1323	#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_pfnCImpl, a0, a1) \
1324	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
1325
1326	/**
1327	* Defers the rest of the instruction emulation to a C implementation routine
1328	* and returns, taking three arguments in addition to the standard ones.
1329	*
1330	* @param a_fFlags IEM_CIMPL_F_XXX.
1331	* @param a_pfnCImpl The pointer to the C routine.
1332	* @param a0 The first extra argument.
1333	* @param a1 The second extra argument.
1334	* @param a2 The third extra argument.
1335	*/
1336	#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_pfnCImpl, a0, a1, a2) \
1337	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
1338
1339	/**
1340	* Defers the rest of the instruction emulation to a C implementation routine
1341	* and returns, taking four arguments in addition to the standard ones.
1342	*
1343	* @param a_fFlags IEM_CIMPL_F_XXX.
1344	* @param a_pfnCImpl The pointer to the C routine.
1345	* @param a0 The first extra argument.
1346	* @param a1 The second extra argument.
1347	* @param a2 The third extra argument.
1348	* @param a3 The fourth extra argument.
1349	*/
1350	#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_pfnCImpl, a0, a1, a2, a3) \
1351	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
1352
1353	/**
1354	* Defers the rest of the instruction emulation to a C implementation routine
1355	* and returns, taking two arguments in addition to the standard ones.
1356	*
1357	* @param a_fFlags IEM_CIMPL_F_XXX.
1358	* @param a_pfnCImpl The pointer to the C routine.
1359	* @param a0 The first extra argument.
1360	* @param a1 The second extra argument.
1361	* @param a2 The third extra argument.
1362	* @param a3 The fourth extra argument.
1363	* @param a4 The fifth extra argument.
1364	*/
1365	#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_pfnCImpl, a0, a1, a2, a3, a4) \
1366	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
1367
1368	/**
1369	* Defers the entire instruction emulation to a C implementation routine and
1370	* returns, only taking the standard parameters.
1371	*
1372	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
1373	*
1374	* @param a_fFlags IEM_CIMPL_F_XXX.
1375	* @param a_pfnCImpl The pointer to the C routine.
1376	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
1377	*/
1378	#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_pfnCImpl) \
1379	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
1380
1381	/**
1382	* Defers the entire instruction emulation to a C implementation routine and
1383	* returns, taking one argument in addition to the standard ones.
1384	*
1385	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
1386	*
1387	* @param a_fFlags IEM_CIMPL_F_XXX.
1388	* @param a_pfnCImpl The pointer to the C routine.
1389	* @param a0 The argument.
1390	*/
1391	#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_pfnCImpl, a0) \
1392	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
1393
1394	/**
1395	* Defers the entire instruction emulation to a C implementation routine and
1396	* returns, taking two arguments in addition to the standard ones.
1397	*
1398	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
1399	*
1400	* @param a_fFlags IEM_CIMPL_F_XXX.
1401	* @param a_pfnCImpl The pointer to the C routine.
1402	* @param a0 The first extra argument.
1403	* @param a1 The second extra argument.
1404	*/
1405	#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_pfnCImpl, a0, a1) \
1406	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
1407
1408	/**
1409	* Defers the entire instruction emulation to a C implementation routine and
1410	* returns, taking three arguments in addition to the standard ones.
1411	*
1412	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
1413	*
1414	* @param a_fFlags IEM_CIMPL_F_XXX.
1415	* @param a_pfnCImpl The pointer to the C routine.
1416	* @param a0 The first extra argument.
1417	* @param a1 The second extra argument.
1418	* @param a2 The third extra argument.
1419	*/
1420	#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_pfnCImpl, a0, a1, a2) \
1421	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
1422
1423
1424	/**
1425	* Calls a FPU assembly implementation taking one visible argument.
1426	*
1427	* @param a_pfnAImpl Pointer to the assembly FPU routine.
1428	* @param a0 The first extra argument.
1429	*/
1430	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
1431	do { \
1432	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
1433	} while (0)
1434
1435	/**
1436	* Calls a FPU assembly implementation taking two visible arguments.
1437	*
1438	* @param a_pfnAImpl Pointer to the assembly FPU routine.
1439	* @param a0 The first extra argument.
1440	* @param a1 The second extra argument.
1441	*/
1442	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
1443	do { \
1444	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
1445	} while (0)
1446
1447	/**
1448	* Calls a FPU assembly implementation taking three visible arguments.
1449	*
1450	* @param a_pfnAImpl Pointer to the assembly FPU routine.
1451	* @param a0 The first extra argument.
1452	* @param a1 The second extra argument.
1453	* @param a2 The third extra argument.
1454	*/
1455	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
1456	do { \
1457	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
1458	} while (0)
1459
1460	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
1461	do { \
1462	(a_FpuData).FSW = (a_FSW); \
1463	(a_FpuData).r80Result = *(a_pr80Value); \
1464	} while (0)
1465
1466	/** Pushes FPU result onto the stack. */
1467	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
1468	iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
1469	/** Pushes FPU result onto the stack and sets the FPUDP. */
1470	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
1471	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
1472
1473	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
1474	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
1475	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
1476
1477	/** Stores FPU result in a stack register. */
1478	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
1479	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
1480	/** Stores FPU result in a stack register and pops the stack. */
1481	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
1482	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
1483	/** Stores FPU result in a stack register and sets the FPUDP. */
1484	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
1485	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
1486	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
1487	* stack. */
1488	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
1489	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
1490
1491	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
1492	#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
1493	iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
1494	/** Free a stack register (for FFREE and FFREEP). */
1495	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
1496	iemFpuStackFree(pVCpu, a_iStReg)
1497	/** Increment the FPU stack pointer. */
1498	#define IEM_MC_FPU_STACK_INC_TOP() \
1499	iemFpuStackIncTop(pVCpu)
1500	/** Decrement the FPU stack pointer. */
1501	#define IEM_MC_FPU_STACK_DEC_TOP() \
1502	iemFpuStackDecTop(pVCpu)
1503
1504	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
1505	#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
1506	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
1507	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
1508	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
1509	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
1510	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
1511	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
1512	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
1513	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
1514	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
1515	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
1516	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
1517	* stack. */
1518	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
1519	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
1520	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
1521	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
1522	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
1523
1524	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
1525	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
1526	iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
1527	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
1528	* stack. */
1529	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
1530	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
1531	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
1532	* FPUDS. */
1533	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
1534	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
1535	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
1536	* FPUDS. Pops stack. */
1537	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
1538	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
1539	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
1540	* stack twice. */
1541	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
1542	iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
1543	/** Raises a FPU stack underflow exception for an instruction pushing a result
1544	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
1545	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
1546	iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
1547	/** Raises a FPU stack underflow exception for an instruction pushing a result
1548	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
1549	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
1550	iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
1551
1552	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
1553	* FPUIP, FPUCS and FOP. */
1554	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
1555	iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
1556	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
1557	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
1558	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
1559	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
1560	/** Prepares for using the FPU state.
1561	* Ensures that we can use the host FPU in the current context (RC+R0.
1562	* Ensures the guest FPU state in the CPUMCTX is up to date. */
1563	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
1564	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
1565	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
1566	/** Actualizes the guest FPU state so it can be accessed and modified. */
1567	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
1568
1569	/** Stores SSE SIMD result updating MXCSR. */
1570	#define IEM_MC_STORE_SSE_RESULT(a_SseData, a_iXmmReg) \
1571	iemSseStoreResult(pVCpu, &a_SseData, a_iXmmReg)
1572	/** Updates MXCSR. */
1573	#define IEM_MC_SSE_UPDATE_MXCSR(a_fMxcsr) \
1574	iemSseUpdateMxcsr(pVCpu, a_fMxcsr)
1575
1576	/** Prepares for using the SSE state.
1577	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
1578	* Ensures the guest SSE state in the CPUMCTX is up to date. */
1579	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
1580	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
1581	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
1582	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
1583	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
1584
1585	/** Prepares for using the AVX state.
1586	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
1587	* Ensures the guest AVX state in the CPUMCTX is up to date.
1588	* @note This will include the AVX512 state too when support for it is added
1589	* due to the zero extending feature of VEX instruction. */
1590	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
1591	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
1592	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
1593	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
1594	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
1595
1596	/**
1597	* Calls a MMX assembly implementation taking two visible arguments.
1598	*
1599	* @param a_pfnAImpl Pointer to the assembly MMX routine.
1600	* @param a0 The first extra argument.
1601	* @param a1 The second extra argument.
1602	*/
1603	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
1604	do { \
1605	IEM_MC_PREPARE_FPU_USAGE(); \
1606	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
1607	} while (0)
1608
1609	/**
1610	* Calls a MMX assembly implementation taking three visible arguments.
1611	*
1612	* @param a_pfnAImpl Pointer to the assembly MMX routine.
1613	* @param a0 The first extra argument.
1614	* @param a1 The second extra argument.
1615	* @param a2 The third extra argument.
1616	*/
1617	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
1618	do { \
1619	IEM_MC_PREPARE_FPU_USAGE(); \
1620	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
1621	} while (0)
1622
1623
1624	/**
1625	* Calls a SSE assembly implementation taking two visible arguments.
1626	*
1627	* @param a_pfnAImpl Pointer to the assembly SSE routine.
1628	* @param a0 The first extra argument.
1629	* @param a1 The second extra argument.
1630	*/
1631	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
1632	do { \
1633	IEM_MC_PREPARE_SSE_USAGE(); \
1634	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
1635	} while (0)
1636
1637	/**
1638	* Calls a SSE assembly implementation taking three visible arguments.
1639	*
1640	* @param a_pfnAImpl Pointer to the assembly SSE routine.
1641	* @param a0 The first extra argument.
1642	* @param a1 The second extra argument.
1643	* @param a2 The third extra argument.
1644	*/
1645	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
1646	do { \
1647	IEM_MC_PREPARE_SSE_USAGE(); \
1648	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
1649	} while (0)
1650
1651
1652	/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
1653	* IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ... */
1654	#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
1655	IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, &pVCpu->cpum.GstCtx.XState, 0)
1656
1657	/**
1658	* Calls a AVX assembly implementation taking two visible arguments.
1659	*
1660	* There is one implicit zero'th argument, a pointer to the extended state.
1661	*
1662	* @param a_pfnAImpl Pointer to the assembly AVX routine.
1663	* @param a1 The first extra argument.
1664	* @param a2 The second extra argument.
1665	*/
1666	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
1667	do { \
1668	IEM_MC_PREPARE_AVX_USAGE(); \
1669	a_pfnAImpl(pXState, (a1), (a2)); \
1670	} while (0)
1671
1672	/**
1673	* Calls a AVX assembly implementation taking three visible arguments.
1674	*
1675	* There is one implicit zero'th argument, a pointer to the extended state.
1676	*
1677	* @param a_pfnAImpl Pointer to the assembly AVX routine.
1678	* @param a1 The first extra argument.
1679	* @param a2 The second extra argument.
1680	* @param a3 The third extra argument.
1681	*/
1682	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
1683	do { \
1684	IEM_MC_PREPARE_AVX_USAGE(); \
1685	a_pfnAImpl(pXState, (a1), (a2), (a3)); \
1686	} while (0)
1687
1688	/** @note Not for IOPL or IF testing. */
1689	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
1690	/** @note Not for IOPL or IF testing. */
1691	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
1692	/** @note Not for IOPL or IF testing. */
1693	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
1694	/** @note Not for IOPL or IF testing. */
1695	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
1696	/** @note Not for IOPL or IF testing. */
1697	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
1698	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
1699	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
1700	/** @note Not for IOPL or IF testing. */
1701	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
1702	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
1703	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
1704	/** @note Not for IOPL or IF testing. */
1705	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
1706	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
1707	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
1708	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
1709	/** @note Not for IOPL or IF testing. */
1710	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
1711	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
1712	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
1713	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
1714	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
1715	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
1716	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
1717	/** @note Not for IOPL or IF testing. */
1718	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
1719	if ( pVCpu->cpum.GstCtx.cx != 0 \
1720	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1721	/** @note Not for IOPL or IF testing. */
1722	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
1723	if ( pVCpu->cpum.GstCtx.ecx != 0 \
1724	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1725	/** @note Not for IOPL or IF testing. */
1726	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
1727	if ( pVCpu->cpum.GstCtx.rcx != 0 \
1728	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1729	/** @note Not for IOPL or IF testing. */
1730	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
1731	if ( pVCpu->cpum.GstCtx.cx != 0 \
1732	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1733	/** @note Not for IOPL or IF testing. */
1734	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
1735	if ( pVCpu->cpum.GstCtx.ecx != 0 \
1736	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1737	/** @note Not for IOPL or IF testing. */
1738	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
1739	if ( pVCpu->cpum.GstCtx.rcx != 0 \
1740	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
1741	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
1742	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
1743
1744	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
1745	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
1746	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
1747	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
1748	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
1749	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
1750	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
1751	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
1752	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
1753	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
1754	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
1755	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
1756	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
1757	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
1758	#define IEM_MC_IF_FCW_IM() \
1759	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
1760	#define IEM_MC_IF_MXCSR_XCPT_PENDING() \
1761	if (( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
1762	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) != 0) {
1763
1764	#define IEM_MC_ELSE() } else {
1765	#define IEM_MC_ENDIF() } do {} while (0)
1766
1767	/** @} */
1768
1769	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
1770

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 100591

Download in other formats: