IEMN8veRecompiler.h@ 106319

Last change on this file since 106319 was 106319, checked in by vboxsync, 2 months ago
VMM/IEM: Reduced parameter count for iemNativeRegAllocTmp*. bugref:10720
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 159.3 KB

Line
1	/* $Id: IEMN8veRecompiler.h 106319 2024-10-15 08:50:24Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43	* Enables generating internal debug info for better TB disassembly dumping. */
44	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING) \|\| 0
45	# define IEMNATIVE_WITH_TB_DEBUG_INFO
46	#endif
47
48	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49	* Enables liveness analysis. */
50	#if 1 \|\| defined(DOXYGEN_RUNNING)
51	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52	#endif
53
54	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
55	* Enables skipping EFLAGS calculations/updating based on liveness info. */
56	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
57	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
58	#endif
59
60	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
61	* Enables strict consistency checks around EFLAGS skipping.
62	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
63	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
64	# ifdef VBOX_STRICT
65	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
66	# endif
67	#elif defined(DOXYGEN_RUNNING)
68	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
69	#endif
70
71	/** @def IEMNATIVE_WITH_EFLAGS_POSTPONING
72	* Enables delaying EFLAGS calculations/updating to conditional code paths
73	* that are (hopefully) not taken so frequently.
74	*
75	* This can only help with case where there is an conditional
76	* call/exception/tbexit that needs the flag, but in the default code stream the
77	* flag will be clobbered. Useful for TlbMiss scenarios and sequences of memory
78	* based instructions clobbering status flags. */
79	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
80	# if 1 \|\| defined(DOXYGEN_RUNNING)
81	# define IEMNATIVE_WITH_EFLAGS_POSTPONING
82	# endif
83	#endif
84	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
85	# ifndef IEMNATIVE_WITH_EFLAGS_SKIPPING
86	# error "IEMNATIVE_WITH_EFLAGS_POSTPONING requires IEMNATIVE_WITH_EFLAGS_SKIPPING at present"
87	# endif
88	#endif
89
90	/** @def IEMLIVENESS_EXTENDED_LAYOUT
91	* Enables the extended liveness data layout. */
92	#if defined(IEMNATIVE_WITH_EFLAGS_POSTPONING) \|\| defined(DOXYGEN_RUNNING) \|\| 0
93	# define IEMLIVENESS_EXTENDED_LAYOUT
94	#endif
95
96
97	#ifdef VBOX_WITH_STATISTICS
98	/** Always count instructions for now. */
99	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
100	#endif
101
102
103	/** @name Stack Frame Layout
104	*
105	* @{ */
106	/** The size of the area for stack variables and spills and stuff.
107	* @note This limit is duplicated in the python script(s). We add 0x40 for
108	* alignment padding. */
109	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
110	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
111	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
112	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
113
114	#ifdef RT_ARCH_AMD64
115	/** An stack alignment adjustment (between non-volatile register pushes and
116	* the stack variable area, so the latter better aligned). */
117	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
118
119	/** Number of stack arguments slots for calls made from the frame. */
120	# ifdef RT_OS_WINDOWS
121	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
122	# else
123	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
124	# endif
125	/** Number of any shadow arguments (spill area) for calls we make. */
126	# ifdef RT_OS_WINDOWS
127	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
128	# else
129	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
130	# endif
131
132	/** Frame pointer (RBP) relative offset of the last push. */
133	# ifdef RT_OS_WINDOWS
134	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
135	# else
136	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
137	# endif
138	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
139	* address for it). */
140	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
141	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
142	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
143	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
144	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
145	# ifdef RT_OS_WINDOWS
146	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
147	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
148	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
149	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
150	# endif
151
152	# ifdef RT_OS_WINDOWS
153	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
154	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
155	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
156	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
157	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
158	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
159	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
160	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
161	# endif
162
163	#elif RT_ARCH_ARM64
164	/** No alignment padding needed for arm64. */
165	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
166	/** No stack argument slots, got 8 registers for arguments will suffice. */
167	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
168	/** There are no argument spill area. */
169	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
170
171	/** Number of saved registers at the top of our stack frame.
172	* This includes the return address and old frame pointer, so x19 thru x30. */
173	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
174	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
175	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
176
177	/** Frame pointer (BP) relative offset of the last push. */
178	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
179
180	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
181	* address for it). */
182	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
183
184	#else
185	# error "port me"
186	#endif
187	/** @} */
188
189
190	/** @name Fixed Register Allocation(s)
191	* @{ */
192	/** @def IEMNATIVE_REG_FIXED_PVMCPU
193	* The number of the register holding the pVCpu pointer. */
194	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
195	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
196	* @note This not available on AMD64, only ARM64. */
197	/** @def IEMNATIVE_REG_FIXED_TMP0
198	* Dedicated temporary register.
199	* @note This has extremely short lifetime, must be used with great care to make
200	* sure any calling code or code being called is making use of it.
201	* It will definitely not survive a call or anything of that nature.
202	* @todo replace this by a register allocator and content tracker. */
203	/** @def IEMNATIVE_REG_FIXED_MASK
204	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
205	* architecture. */
206	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
207	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
208	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
209	* architecture. */
210	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
211	* Dedicated temporary SIMD register. */
212	#endif
213	#if defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
214	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
215	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
216	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
217	# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
218	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
219	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
220	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
221	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
222	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
223	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
224	# else
225	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
226	# endif
227	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
228	\| RT_BIT_32(ARMV8_A64_REG_LR) \
229	\| RT_BIT_32(ARMV8_A64_REG_BP) \
230	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
231	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
232	\| RT_BIT_32(ARMV8_A64_REG_X18) \
233	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
234	\| IEMNATIVE_REG_FIXED_MASK_ADD)
235
236	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
237	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
238	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
239	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
240	# else
241	/** @note
242	* ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
243	* support emulating 256-bit registers we pair two real registers statically to
244	* one virtual for now, leaving us with only 16 256-bit registers. We always
245	* pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
246	* the register allocator assumes that it will be always free when the lower is
247	* picked.
248	*
249	* Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
250	* touch them in order to avoid having to save and restore them in the
251	* prologue/epilogue.
252	*/
253	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
254	\| RT_BIT_32(ARMV8_A64_REG_Q31) \
255	\| RT_BIT_32(ARMV8_A64_REG_Q30) \
256	\| RT_BIT_32(ARMV8_A64_REG_Q29) \
257	\| RT_BIT_32(ARMV8_A64_REG_Q27) \
258	\| RT_BIT_32(ARMV8_A64_REG_Q25) \
259	\| RT_BIT_32(ARMV8_A64_REG_Q23) \
260	\| RT_BIT_32(ARMV8_A64_REG_Q21) \
261	\| RT_BIT_32(ARMV8_A64_REG_Q19) \
262	\| RT_BIT_32(ARMV8_A64_REG_Q17) \
263	\| RT_BIT_32(ARMV8_A64_REG_Q15) \
264	\| RT_BIT_32(ARMV8_A64_REG_Q13) \
265	\| RT_BIT_32(ARMV8_A64_REG_Q11) \
266	\| RT_BIT_32(ARMV8_A64_REG_Q9) \
267	\| RT_BIT_32(ARMV8_A64_REG_Q7) \
268	\| RT_BIT_32(ARMV8_A64_REG_Q5) \
269	\| RT_BIT_32(ARMV8_A64_REG_Q3) \
270	\| RT_BIT_32(ARMV8_A64_REG_Q1))
271	# endif
272	# endif
273
274	#elif defined(RT_ARCH_AMD64)
275	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
276	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
277	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
278	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
279	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
280	\| RT_BIT_32(X86_GREG_xSP) \
281	\| RT_BIT_32(X86_GREG_xBP) )
282
283	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
284	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
285	# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
286	# ifndef _MSC_VER
287	# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
288	# endif
289	# endif
290	# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
291	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
292	# else
293	/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
294	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
295	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
296	# endif
297	# endif
298
299	#else
300	# error "port me"
301	#endif
302	/** @} */
303
304	/** @name Call related registers.
305	* @{ */
306	/** @def IEMNATIVE_CALL_RET_GREG
307	* The return value register. */
308	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
309	* Number of arguments in registers. */
310	/** @def IEMNATIVE_CALL_ARG0_GREG
311	* The general purpose register carrying argument \#0. */
312	/** @def IEMNATIVE_CALL_ARG1_GREG
313	* The general purpose register carrying argument \#1. */
314	/** @def IEMNATIVE_CALL_ARG2_GREG
315	* The general purpose register carrying argument \#2. */
316	/** @def IEMNATIVE_CALL_ARG3_GREG
317	* The general purpose register carrying argument \#3. */
318	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
319	* Mask of registers the callee will not save and may trash. */
320	#ifdef RT_ARCH_AMD64
321	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
322
323	# ifdef RT_OS_WINDOWS
324	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
325	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
326	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
327	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
328	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
329	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
330	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
331	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
332	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
333	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
334	\| RT_BIT_32(X86_GREG_xCX) \
335	\| RT_BIT_32(X86_GREG_xDX) \
336	\| RT_BIT_32(X86_GREG_x8) \
337	\| RT_BIT_32(X86_GREG_x9) \
338	\| RT_BIT_32(X86_GREG_x10) \
339	\| RT_BIT_32(X86_GREG_x11) )
340	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
341	/* xmm0 - xmm5 are marked as volatile. */
342	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
343	# endif
344
345	# else /* !RT_OS_WINDOWS */
346	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
347	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
348	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
349	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
350	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
351	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
352	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
353	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
354	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
355	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
356	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
357	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
358	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
359	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
360	\| RT_BIT_32(X86_GREG_xCX) \
361	\| RT_BIT_32(X86_GREG_xDX) \
362	\| RT_BIT_32(X86_GREG_xDI) \
363	\| RT_BIT_32(X86_GREG_xSI) \
364	\| RT_BIT_32(X86_GREG_x8) \
365	\| RT_BIT_32(X86_GREG_x9) \
366	\| RT_BIT_32(X86_GREG_x10) \
367	\| RT_BIT_32(X86_GREG_x11) )
368	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
369	/* xmm0 - xmm15 are marked as volatile. */
370	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
371	# endif
372	# endif /* !RT_OS_WINDOWS */
373
374	#elif defined(RT_ARCH_ARM64)
375	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
376	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
377	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
378	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
379	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
380	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
381	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
382	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
383	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
384	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
385	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
386	\| RT_BIT_32(ARMV8_A64_REG_X1) \
387	\| RT_BIT_32(ARMV8_A64_REG_X2) \
388	\| RT_BIT_32(ARMV8_A64_REG_X3) \
389	\| RT_BIT_32(ARMV8_A64_REG_X4) \
390	\| RT_BIT_32(ARMV8_A64_REG_X5) \
391	\| RT_BIT_32(ARMV8_A64_REG_X6) \
392	\| RT_BIT_32(ARMV8_A64_REG_X7) )
393	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
394	\| RT_BIT_32(ARMV8_A64_REG_X1) \
395	\| RT_BIT_32(ARMV8_A64_REG_X2) \
396	\| RT_BIT_32(ARMV8_A64_REG_X3) \
397	\| RT_BIT_32(ARMV8_A64_REG_X4) \
398	\| RT_BIT_32(ARMV8_A64_REG_X5) \
399	\| RT_BIT_32(ARMV8_A64_REG_X6) \
400	\| RT_BIT_32(ARMV8_A64_REG_X7) \
401	\| RT_BIT_32(ARMV8_A64_REG_X8) \
402	\| RT_BIT_32(ARMV8_A64_REG_X9) \
403	\| RT_BIT_32(ARMV8_A64_REG_X10) \
404	\| RT_BIT_32(ARMV8_A64_REG_X11) \
405	\| RT_BIT_32(ARMV8_A64_REG_X12) \
406	\| RT_BIT_32(ARMV8_A64_REG_X13) \
407	\| RT_BIT_32(ARMV8_A64_REG_X14) \
408	\| RT_BIT_32(ARMV8_A64_REG_X15) \
409	\| RT_BIT_32(ARMV8_A64_REG_X16) \
410	\| RT_BIT_32(ARMV8_A64_REG_X17) )
411	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
412	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
413	* so to simplify our life a bit we just mark everything as volatile. */
414	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
415	# endif
416
417	#endif
418
419	/** This is the maximum argument count we'll ever be needing. */
420	#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
421	#ifdef RT_OS_WINDOWS
422	# ifdef VBOXSTRICTRC_STRICT_ENABLED
423	# undef IEMNATIVE_CALL_MAX_ARG_COUNT
424	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
425	# endif
426	#endif
427
428	/** @def IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK
429	* Variant of IEMNATIVE_CALL_VOLATILE_GREG_MASK that excludes
430	* IEMNATIVE_REG_FIXED_TMP0 on hosts that uses it. */
431	#ifdef IEMNATIVE_REG_FIXED_TMP0
432	# ifdef IEMNATIVE_REG_FIXED_TMP1
433	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK ( IEMNATIVE_CALL_VOLATILE_GREG_MASK \
434	& ~( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
435	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1)))
436	# else
437	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK (IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0))
438	# endif
439	#else
440	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK IEMNATIVE_CALL_VOLATILE_GREG_MASK
441	#endif
442
443	/** @def IEMNATIVE_CALL_NONVOLATILE_GREG_MASK
444	* The allocatable non-volatile general purpose register set. */
445	#define IEMNATIVE_CALL_NONVOLATILE_GREG_MASK \
446	(~IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK & IEMNATIVE_HST_GREG_MASK)
447	/** @} */
448
449
450	/** @def IEMNATIVE_HST_GREG_COUNT
451	* Number of host general purpose registers we tracker. */
452	/** @def IEMNATIVE_HST_GREG_MASK
453	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
454	* inverted register masks and such to get down to a correct set of regs. */
455	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
456	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
457	* Number of host SIMD registers we track. */
458	/** @def IEMNATIVE_HST_SIMD_REG_MASK
459	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
460	* inverted register masks and such to get down to a correct set of regs. */
461	#endif
462	#ifdef RT_ARCH_AMD64
463	# define IEMNATIVE_HST_GREG_COUNT 16
464	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
465
466	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
467	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
468	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
469	# endif
470
471	#elif defined(RT_ARCH_ARM64)
472	# define IEMNATIVE_HST_GREG_COUNT 32
473	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
474
475	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
476	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
477	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
478	# endif
479
480	#else
481	# error "Port me!"
482	#endif
483
484
485	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
486
487
488	/** Native code generator label types. */
489	typedef enum
490	{
491	kIemNativeLabelType_Invalid = 0,
492	/** @name Exit reasons - Labels w/o data, only once instance per TB.
493	*
494	* The labels requiring register inputs are documented.
495	*
496	* @note Jumps to these requires instructions that are capable of spanning the
497	* max TB length.
498	* @{
499	*/
500	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
501	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
502	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
503	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
504	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
505	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
506	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
507	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
508	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
509	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
510	kIemNativeLabelType_ObsoleteTb, /*< Calls iemNativeHlpObsoleteTb (no inputs). /
511	kIemNativeLabelType_NeedCsLimChecking, /*< Calls iemNativeHlpNeedCsLimChecking (no inputs). /
512	kIemNativeLabelType_CheckBranchMiss, /*< Calls iemNativeHlpCheckBranchMiss (no inputs). /
513	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
514
515	/* Manually defined labels: */
516	/*< Returns with VINF_SUCCESS, no inputs. /
517	kIemNativeLabelType_ReturnSuccess,
518	/** Returns with VINF_IEM_REEXEC_FINISH_WITH_FLAGS, no inputs. */
519	kIemNativeLabelType_ReturnWithFlags,
520	/** Returns with VINF_IEM_REEXEC_BREAK, no inputs. */
521	kIemNativeLabelType_ReturnBreak,
522	/** Returns with VINF_IEM_REEXEC_BREAK_FF, no inputs. */
523	kIemNativeLabelType_ReturnBreakFF,
524	/** The last TB exit label that doesn't have any input registers. */
525	kIemNativeLabelType_LastTbExitWithoutInputs = kIemNativeLabelType_ReturnBreakFF,
526
527	/** Argument registers 1, 2 & 3 are set up. */
528	kIemNativeLabelType_ReturnBreakViaLookup,
529	/** Argument registers 1, 2 & 3 are set up. */
530	kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
531	/** Argument registers 1 & 2 are set up. */
532	kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
533	/** Argument registers 1 & 2 are set up. */
534	kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
535	/** Return register holds the RC and the instruction number is in CL/RCX
536	* on amd64 and the 2rd argument register elsewhere. */
537	kIemNativeLabelType_NonZeroRetOrPassUp,
538
539	/** The last fixup for branches that can span almost the whole TB length.
540	* @note Whether kIemNativeLabelType_Return needs to be one of these is
541	* a bit questionable, since nobody jumps to it except other tail code. */
542	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
543	/** The last fixup for branches that exits the TB. */
544	kIemNativeLabelType_LastTbExit = kIemNativeLabelType_NonZeroRetOrPassUp,
545	/** @} */
546
547	/** Loop-jump target. */
548	kIemNativeLabelType_LoopJumpTarget,
549
550	/*
551	* Labels with data, potentially multiple instances per TB:
552	*
553	* These are localized labels, so no fixed jump type restrictions here.
554	*/
555	kIemNativeLabelType_FirstWithMultipleInstances,
556	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
557	kIemNativeLabelType_Else,
558	kIemNativeLabelType_Endif,
559	kIemNativeLabelType_CheckIrq,
560	kIemNativeLabelType_TlbLookup,
561	kIemNativeLabelType_TlbMiss,
562	kIemNativeLabelType_TlbDone,
563	kIemNativeLabelType_End
564	} IEMNATIVELABELTYPE;
565
566	#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
567	((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
568
569	#define IEMNATIVELABELTYPE_IS_EXIT_WITHOUT_INPUTS(a_enmLabel) \
570	((a_enmLabel) <= kIemNativeLabelType_LastTbExitWithoutInputs && (a_enmLabel) > kIemNativeLabelType_Invalid)
571
572	/**
573	* Get the mask of input registers for an TB exit label.
574	* This will return zero for any non-exit lable.
575	*/
576	#ifdef RT_ARCH_AMD64
577	# define IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmLabel) \
578	( (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookup \
579	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithIrq \
580	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
581	: (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlb \
582	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq \
583	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
584	: (a_enmLabel) == kIemNativeLabelType_NonZeroRetOrPassUp \
585	? RT_BIT_32(IEMNATIVE_CALL_RET_GREG) \| RT_BIT_32(X86_GREG_xCX) /* <-- the difference */ \
586	: 0)
587	# else
588	# define IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmLabel) \
589	( (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookup \
590	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithIrq \
591	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
592	: (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlb \
593	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq \
594	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
595	: (a_enmLabel) == kIemNativeLabelType_NonZeroRetOrPassUp \
596	? RT_BIT_32(IEMNATIVE_CALL_RET_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
597	: 0)
598	#endif
599
600
601	/** Native code generator label definition. */
602	typedef struct IEMNATIVELABEL
603	{
604	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
605	* the epilog. */
606	uint32_t off;
607	/** The type of label (IEMNATIVELABELTYPE). */
608	uint16_t enmType;
609	/** Additional label data, type specific. */
610	uint16_t uData;
611	} IEMNATIVELABEL;
612	/** Pointer to a label. */
613	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
614
615
616
617	/** Native code generator fixup types. */
618	typedef enum
619	{
620	kIemNativeFixupType_Invalid = 0,
621	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
622	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
623	kIemNativeFixupType_Rel32,
624	#elif defined(RT_ARCH_ARM64)
625	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
626	kIemNativeFixupType_RelImm26At0,
627	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
628	kIemNativeFixupType_RelImm19At5,
629	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
630	kIemNativeFixupType_RelImm14At5,
631	#endif
632	kIemNativeFixupType_End
633	} IEMNATIVEFIXUPTYPE;
634
635	/** Native code generator fixup. */
636	typedef struct IEMNATIVEFIXUP
637	{
638	/** Code offset of the fixup location. */
639	uint32_t off;
640	/** The IEMNATIVELABEL this is a fixup for. */
641	uint16_t idxLabel;
642	/** The fixup type (IEMNATIVEFIXUPTYPE). */
643	uint8_t enmType;
644	/** Addend or other data. */
645	int8_t offAddend;
646	} IEMNATIVEFIXUP;
647	/** Pointer to a native code generator fixup. */
648	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
649
650
651
652	/** Native code generator fixup to per chunk TB tail code. */
653	typedef struct IEMNATIVEEXITFIXUP
654	{
655	/** Code offset of the fixup location. */
656	uint32_t off;
657	/** The exit reason. */
658	IEMNATIVELABELTYPE enmExitReason;
659	} IEMNATIVEEXITFIXUP;
660	/** Pointer to a native code generator TB exit fixup. */
661	typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
662
663	/**
664	* Per executable memory chunk context with addresses for common code.
665	*/
666	typedef struct IEMNATIVEPERCHUNKCTX
667	{
668	/** Pointers to the exit labels */
669	PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
670	} IEMNATIVEPERCHUNKCTX;
671	/** Pointer to per-chunk recompiler context. */
672	typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
673	/** Pointer to const per-chunk recompiler context. */
674	typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
675
676
677
678	/**
679	* One bit of the state.
680	*
681	* Each register state takes up two bits. We keep the two bits in two separate
682	* 64-bit words to simplify applying them to the guest shadow register mask in
683	* the register allocator.
684	*/
685	typedef union IEMLIVENESSBIT
686	{
687	uint64_t bm64;
688	RT_GCC_EXTENSION struct
689	{ /* bit no */
690	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
691	uint64_t fCr0 : 1; /*< 0x10 / 16: /
692	uint64_t fCr4 : 1; /*< 0x11 / 17: /
693	uint64_t fFcw : 1; /*< 0x12 / 18: /
694	uint64_t fFsw : 1; /*< 0x13 / 19: /
695	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
696	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
697	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
698	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
699	uint64_t fXcr0 : 1; /*< 0x2c / 44: /
700	uint64_t fMxCsr : 1; /*< 0x2d / 45: /
701	uint64_t fEflOther : 1; /*< 0x2e / 46: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
702	uint64_t fEflCf : 1; /*< 0x2f / 47: Carry flag (X86_EFL_CF / 0). /
703	uint64_t fEflPf : 1; /*< 0x30 / 48: Parity flag (X86_EFL_PF / 2). /
704	uint64_t fEflAf : 1; /*< 0x31 / 59: Auxilary carry flag (X86_EFL_AF / 4). /
705	uint64_t fEflZf : 1; /*< 0x32 / 50: Zero flag (X86_EFL_ZF / 6). /
706	uint64_t fEflSf : 1; /*< 0x33 / 51: Signed flag (X86_EFL_SF / 7). /
707	uint64_t fEflOf : 1; /*< 0x34 / 52: Overflow flag (X86_EFL_OF / 12). /
708	uint64_t fUnusedPc : 1; /*< 0x35 / 53: (PC in ) /
709	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
710	};
711	} IEMLIVENESSBIT;
712	AssertCompileSize(IEMLIVENESSBIT, 8);
713
714	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
715	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
716	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
717	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
718	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
719	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
720	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
721	#define IEMLIVENESSBIT_IDX_EFL_COUNT 7
722
723
724	/**
725	* A liveness state entry.
726	*
727	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
728	* Once we add a SSE register shadowing, we'll add another 64-bit element for
729	* that.
730	*/
731	typedef union IEMLIVENESSENTRY
732	{
733	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
734	uint64_t bm64[16 / 8];
735	uint16_t bm32[16 / 4];
736	uint16_t bm16[16 / 2];
737	uint8_t bm8[ 16 / 1];
738	IEMLIVENESSBIT aBits[2];
739	#else
740	uint64_t bm64[32 / 8];
741	uint16_t bm32[32 / 4];
742	uint16_t bm16[32 / 2];
743	uint8_t bm8[ 32 / 1];
744	IEMLIVENESSBIT aBits[4];
745	#endif
746	RT_GCC_EXTENSION struct
747	{
748	/** Bit \#0 of the register states. */
749	IEMLIVENESSBIT Bit0;
750	/** Bit \#1 of the register states. */
751	IEMLIVENESSBIT Bit1;
752	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
753	/** Bit \#2 of the register states. */
754	IEMLIVENESSBIT Bit2;
755	/** Bit \#3 of the register states. */
756	IEMLIVENESSBIT Bit3;
757	#endif
758	};
759	} IEMLIVENESSENTRY;
760	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
761	AssertCompileSize(IEMLIVENESSENTRY, 16);
762	#else
763	AssertCompileSize(IEMLIVENESSENTRY, 32);
764	#endif
765	/** Pointer to a liveness state entry. */
766	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
767	/** Pointer to a const liveness state entry. */
768	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
769
770	/** @name 64-bit value masks for IEMLIVENESSENTRY.
771	* @{ / / 0xzzzzyyyyxxxxwwww */
772	#define IEMLIVENESSBIT_MASK UINT64_C(0x001fffffffffffff)
773
774	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
775	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
776	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
777
778	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
779	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
780	#endif
781
782	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x001fc00000000000)
783	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x001f800000000000)
784
785	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
786	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
787	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
788	#endif
789	/** @} */
790
791
792	/** @name The liveness state for a register.
793	*
794	* The state values have been picked to with state accumulation in mind (what
795	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
796	* performance critical work done with the values.
797	*
798	* This is a compressed state that only requires 2 bits per register.
799	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
800	* 1. the incoming state from the following call,
801	* 2. the outgoing state for this call,
802	* 3. mask of the entries set in the 2nd.
803	*
804	* The mask entry (3rd one above) will be used both when updating the outgoing
805	* state and when merging in incoming state for registers not touched by the
806	* current call.
807	*
808	*
809	* Extended Layout:
810	*
811	* The extended layout variation differs from the above as it records the
812	* different register accesses as individual bits, and it is currently used for
813	* the delayed EFLAGS calculation experiments. The latter means that
814	* calls/tb-exits and potential calls/exceptions/tb-exits are recorded
815	* separately so the latter can be checked for in combination with clobbering.
816	*
817	* @{ */
818	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
819	/** The register will be clobbered and the current value thrown away.
820	*
821	* When this is applied to the state (2) we'll simply be AND'ing it with the
822	* (old) mask (3) and adding the register to the mask. This way we'll
823	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
824	* IEMLIVENESS_STATE_INPUT states. */
825	# define IEMLIVENESS_STATE_CLOBBERED 0
826	/** The register is unused in the remainder of the TB.
827	*
828	* This is an initial state and can not be set by any of the
829	* iemNativeLivenessFunc_xxxx callbacks. */
830	# define IEMLIVENESS_STATE_UNUSED 1
831	/** The register value is required in a potential call or exception.
832	*
833	* This means that the register value must be calculated and is best written to
834	* the state, but that any shadowing registers can be flushed thereafter as it's
835	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
836	*
837	* It is typically applied across the board, but we preserve incoming
838	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
839	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
840	* 1. r0 = old & ~mask;
841	* 2. r0 = t1 & (t1 >> 1);
842	* 3. state \|= r0 \| 0b10;
843	* 4. mask = ~0;
844	*/
845	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
846	/** The register value is used as input.
847	*
848	* This means that the register value must be calculated and it is best to keep
849	* it in a register. It does not need to be writtent out as such. This is the
850	* highest priority state.
851	*
852	* Whether the call modifies the register or not isn't relevant to earlier
853	* calls, so that's not recorded.
854	*
855	* When applying this state we just or in the value in the outgoing state and
856	* mask. */
857	# define IEMLIVENESS_STATE_INPUT 3
858	/** Mask of the state bits. */
859	# define IEMLIVENESS_STATE_MASK 3
860	/** The number of bits per state. */
861	# define IEMLIVENESS_STATE_BIT_COUNT 2
862
863	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state.
864	* @note only used in assertions. */
865	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
866	/** Check if we're expecting read accesses to a register with the given (previous) liveness state.
867	* @note only used in assertions. */
868	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
869	/** Check if a register clobbering is expected given the (previous) liveness state.
870	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
871	* include INPUT if the register is used in more than one place.
872	* @note only used in assertions. */
873	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
874
875	/** Check if all status flags are going to be clobbered and doesn't need
876	* calculating in the current step.
877	* @param a_pCurEntry The current liveness entry.
878	* @note Used by actual code. */
879	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
880	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
881
882	/***
883	* Construct a mask of what will be clobbered and never used.
884	*
885	* This is mainly used with IEMLIVENESSBIT_STATUS_EFL_MASK to avoid
886	* unnecessary EFLAGS calculations.
887	*
888	* @param a_pCurEntry The current liveness entry.
889	* @note Used by actual code.
890	*/
891	# define IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(a_pCurEntry) \
892	( ~((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_MASK )
893
894	/** Construct a mask of the guest registers in the UNUSED and XCPT_OR_CALL
895	* states, as these are no longer needed.
896	* @param a_pCurEntry The current liveness entry.
897	* @note Used by actual code. */
898	AssertCompile(IEMLIVENESS_STATE_UNUSED == 1 && IEMLIVENESS_STATE_XCPT_OR_CALL == 2);
899	# define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
900	( (a_pCurEntry)->Bit0.bm64 ^ (a_pCurEntry)->Bit1.bm64 )
901
902
903	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
904	/** The register is not used any more. */
905	# define IEMLIVENESS_STATE_UNUSED 0
906	/** Flag: The register is required in a potential call or/and exception. */
907	# define IEMLIVENESS_STATE_POTENTIAL_CALL 1
908	# define IEMLIVENESS_BIT_POTENTIAL_CALL 0
909	/** Flag: The register is read. */
910	# define IEMLIVENESS_STATE_READ 2
911	# define IEMLIVENESS_BIT_READ 1
912	/** Flag: The register is written. */
913	# define IEMLIVENESS_STATE_WRITE 4
914	# define IEMLIVENESS_BIT_WRITE 2
915	/** Flag: Unconditional call. */
916	# define IEMLIVENESS_STATE_CALL 8
917	# define IEMLIVENESS_BIT_CALL 3
918
919	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
920	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
921	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
922	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
923
924	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
925	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
926	&& !( ( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
927	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
928	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) \
929	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
930
931	/** Construct a mask of the registers not in the read or write state.
932	* @note We could skips writes, if they aren't from us, as this is just a hack
933	* to prevent trashing registers that have just been written or will be
934	* written when we retire the current instruction.
935	* @param a_pCurEntry The current liveness entry.
936	* @note Used by actual code. */
937	# define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
938	( ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
939	& ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
940	& IEMLIVENESSBIT_MASK )
941
942	/***
943	* Construct a mask of what will be clobbered and never used.
944	*
945	* This is mainly used with IEMLIVENESSBIT_STATUS_EFL_MASK to avoid
946	* unnecessary EFLAGS calculations.
947	*
948	* @param a_pCurEntry The current liveness entry.
949	* @note Used by actual code.
950	*/
951	# define IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(a_pCurEntry) \
952	( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
953	& ~( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
954	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
955	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) )
956
957	/**
958	* Construct a mask of what (EFLAGS) which can be postponed.
959	*
960	* The postponement is for the avoiding EFLAGS status bits calculations in the
961	* primary code stream whenever possible, and instead only do these in the TLB
962	* load and TB exit code paths which shouldn't be traveled quite as often.
963	* A requirement, though, is that the status bits will be clobbered later in the
964	* TB.
965	*
966	* User need to apply IEMLIVENESSBIT_STATUS_EFL_MASK if appropriate/necessary.
967	*
968	* @param a_pCurEntry The current liveness entry.
969	* @note Used by actual code.
970	*/
971	# define IEMLIVENESS_STATE_GET_CAN_BE_POSTPONED_SET(a_pCurEntry) \
972	( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
973	& (a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
974	& ~( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
975	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) )
976
977	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
978	/** @} */
979
980	/** @name Liveness helpers for builtin functions and similar.
981	*
982	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
983	* own set of manipulator macros for those.
984	*
985	* @{ */
986	/** Initializing the state as all unused. */
987	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
988	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
989	do { \
990	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
991	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
992	} while (0)
993	#else
994	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
995	do { \
996	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
997	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
998	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
999	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
1000	} while (0)
1001	#endif
1002
1003	/** Initializing the outgoing state with a potential xcpt or call state.
1004	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT.
1005	*
1006	* @note Must invoke IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL when done!
1007	*/
1008	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1009	# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1010	do { \
1011	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
1012	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
1013	} while (0)
1014	#else
1015	# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1016	do { \
1017	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
1018	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
1019	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
1020	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
1021	} while (0)
1022	#endif
1023
1024	/** Completes IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL after applying any
1025	* other state modifications.
1026	*/
1027	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1028	# define IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) ((void)0)
1029	#else
1030	# define IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1031	do { \
1032	uint64_t const fInhMask = ~( (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL].bm64 \
1033	\| (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE].bm64); \
1034	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 & fInhMask; \
1035	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64 & fInhMask; \
1036	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & fInhMask; \
1037	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_CALL].bm64 & fInhMask; \
1038	} while (0)
1039	#endif
1040
1041	/** Initializing the outgoing state with an unconditional call state.
1042	* This should only really be used alone. */
1043	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1044	# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
1045	do { \
1046	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
1047	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
1048	} while (0)
1049	#else
1050	# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
1051	do { \
1052	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
1053	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
1054	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
1055	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
1056	RT_NOREF(a_pIncoming); \
1057	} while (0)
1058	#endif
1059
1060	#if 0 /* unused */
1061	/** Initializing the outgoing state with an unconditional call state as well as
1062	* an potential call/exception preceeding it.
1063	* This should only really be used alone. */
1064	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1065	# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1066	do { \
1067	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
1068	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
1069	} while (0)
1070	#else
1071	# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1072	do { \
1073	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
1074	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
1075	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
1076	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
1077	} while (0)
1078	#endif
1079	#endif
1080
1081	/** Adds a segment base register as input to the outgoing state. */
1082	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1083	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
1084	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
1085	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
1086	} while (0)
1087	#else
1088	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
1089	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
1090	} while (0)
1091	#endif
1092
1093	/** Adds a segment attribute register as input to the outgoing state. */
1094	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1095	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
1096	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
1097	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
1098	} while (0)
1099	#else
1100	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
1101	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
1102	} while (0)
1103	#endif
1104
1105	/** Adds a segment limit register as input to the outgoing state. */
1106	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1107	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
1108	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
1109	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
1110	} while (0)
1111	#else
1112	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
1113	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
1114	} while (0)
1115	#endif
1116
1117	/** Adds a segment limit register as input to the outgoing state. */
1118	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1119	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
1120	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
1121	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
1122	} while (0)
1123	#else
1124	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
1125	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
1126	} while (0)
1127	#endif
1128	/** @} */
1129
1130	/** @def IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY
1131	* Debug assertion that the required flags are available and not incorrectly skipped.
1132	*/
1133	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
1134	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded) \
1135	AssertMsg(!((a_pReNative)->fSkippingEFlags & (a_fEflNeeded)), \
1136	("%#x & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, \
1137	a_fEflNeeded, (a_pReNative)->fSkippingEFlags & (a_fEflNeeded) ))
1138	#else
1139	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded) ((void)0)
1140	#endif
1141
1142	/** @def IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY
1143	* Debug assertion that the required flags are available and not incorrectly postponed.
1144	*/
1145	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1146	# define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) \
1147	AssertMsg(!((a_pReNative)->PostponedEfl.fEFlags & (a_fEflNeeded)), \
1148	("%#x & %#x -> %#x\n", (a_pReNative)->PostponedEfl.fEFlags, \
1149	a_fEflNeeded, (a_pReNative)->PostponedEfl.fEFlags & (a_fEflNeeded) ))
1150	#else
1151	# define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) ((void)0)
1152	#endif
1153
1154	/** @def IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING
1155	* Debug assertion that the required flags are available and not incorrectly
1156	* skipped or postponed.
1157	*/
1158	#if defined(IEMNATIVE_WITH_EFLAGS_SKIPPING) && defined(IEMNATIVE_WITH_EFLAGS_POSTPONING)
1159	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
1160	AssertMsg(!(((a_pReNative)->fSkippingEFlags \| (a_pReNative)->PostponedEfl.fEFlags) & (a_fEflNeeded)), \
1161	("(%#x \| %#x) & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, (a_pReNative)->PostponedEfl.fEFlags, \
1162	a_fEflNeeded, ((a_pReNative)->fSkippingEFlags \| (a_pReNative)->PostponedEfl.fEFlags) & (a_fEflNeeded) ))
1163	#elif defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)
1164	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
1165	IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded)
1166	#elif defined(IEMNATIVE_WITH_EFLAGS_POSTPONING) \
1167	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
1168	IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded)
1169	#else
1170	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) ((void)0)
1171	#endif
1172
1173	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
1174	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
1175	* calculated and up to date. This is to double check that we haven't skipped
1176	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
1177	* @note has to be placed in
1178	*/
1179	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1180	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { \
1181	(a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); \
1182	} while (0)
1183	#else
1184	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
1185	#endif
1186
1187
1188	/** @def IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS
1189	* Number of extra instructions to allocate for each TB exit to account for
1190	* postponed EFLAGS calculations.
1191	*/
1192	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1193	# ifdef RT_ARCH_AMD64
1194	# ifdef VBOX_STRICT
1195	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 64
1196	# else
1197	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 32
1198	# endif
1199	# elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
1200	# ifdef VBOX_STRICT
1201	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 48
1202	# else
1203	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 32
1204	# endif
1205	# else
1206	# error "port me"
1207	# endif
1208	#else
1209	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 0
1210	#endif
1211
1212	/** @def IEMNATIVE_CLEAR_POSTPONED_EFLAGS
1213	* Helper macro function for calling iemNativeClearPostponedEFlags() when
1214	* IEMNATIVE_WITH_EFLAGS_POSTPONING is enabled.
1215	*/
1216	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1217	# define IEMNATIVE_CLEAR_POSTPONED_EFLAGS(a_pReNative, a_fEflClobbered) iemNativeClearPostponedEFlags<a_fEflClobbered>(a_pReNative)
1218	#else
1219	# define IEMNATIVE_CLEAR_POSTPONED_EFLAGS(a_pReNative, a_fEflClobbered) ((void)0)
1220	#endif
1221
1222	/** @def IEMNATIVE_HAS_POSTPONED_EFLAGS_CALCS
1223	* Macro for testing whether there are currently any postponed EFLAGS calcs w/o
1224	* needing to \#ifdef the check.
1225	*/
1226	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1227	# define IEMNATIVE_HAS_POSTPONED_EFLAGS_CALCS(a_pReNative) ((a_pReNative)->PostponedEfl.fEFlags != 0)
1228	#else
1229	# define IEMNATIVE_HAS_POSTPONED_EFLAGS_CALCS(a_pReNative) false
1230	#endif
1231
1232
1233	/**
1234	* Translation block debug info entry type.
1235	*/
1236	typedef enum IEMTBDBGENTRYTYPE
1237	{
1238	kIemTbDbgEntryType_Invalid = 0,
1239	/** The entry is for marking a native code position.
1240	* Entries following this all apply to this position. */
1241	kIemTbDbgEntryType_NativeOffset,
1242	/** The entry is for a new guest instruction. */
1243	kIemTbDbgEntryType_GuestInstruction,
1244	/** Marks the start of a threaded call. */
1245	kIemTbDbgEntryType_ThreadedCall,
1246	/** Marks the location of a label. */
1247	kIemTbDbgEntryType_Label,
1248	/** Info about a host register shadowing a guest register. */
1249	kIemTbDbgEntryType_GuestRegShadowing,
1250	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1251	/** Info about a host SIMD register shadowing a guest SIMD register. */
1252	kIemTbDbgEntryType_GuestSimdRegShadowing,
1253	#endif
1254	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1255	/** Info about a delayed RIP update. */
1256	kIemTbDbgEntryType_DelayedPcUpdate,
1257	#endif
1258	#if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1259	/** Info about a shadowed guest register becoming dirty. */
1260	kIemTbDbgEntryType_GuestRegDirty,
1261	/** Info about register writeback/flush oepration. */
1262	kIemTbDbgEntryType_GuestRegWriteback,
1263	#endif
1264	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1265	/** Info about a delayed EFLAGS calculation. */
1266	kIemTbDbgEntryType_PostponedEFlagsCalc,
1267	#endif
1268	kIemTbDbgEntryType_End
1269	} IEMTBDBGENTRYTYPE;
1270
1271	/**
1272	* Translation block debug info entry.
1273	*/
1274	typedef union IEMTBDBGENTRY
1275	{
1276	/** Plain 32-bit view. */
1277	uint32_t u;
1278
1279	/** Generic view for getting at the type field. */
1280	struct
1281	{
1282	/** IEMTBDBGENTRYTYPE */
1283	uint32_t uType : 4;
1284	uint32_t uTypeSpecific : 28;
1285	} Gen;
1286
1287	struct
1288	{
1289	/** kIemTbDbgEntryType_ThreadedCall1. */
1290	uint32_t uType : 4;
1291	/** Native code offset. */
1292	uint32_t offNative : 28;
1293	} NativeOffset;
1294
1295	struct
1296	{
1297	/** kIemTbDbgEntryType_GuestInstruction. */
1298	uint32_t uType : 4;
1299	uint32_t uUnused : 4;
1300	/** The IEM_F_XXX flags. */
1301	uint32_t fExec : 24;
1302	} GuestInstruction;
1303
1304	struct
1305	{
1306	/* kIemTbDbgEntryType_ThreadedCall. */
1307	uint32_t uType : 4;
1308	/** Set if the call was recompiled to native code, clear if just calling
1309	* threaded function. */
1310	uint32_t fRecompiled : 1;
1311	uint32_t uUnused : 11;
1312	/** The threaded call number (IEMTHREADEDFUNCS). */
1313	uint32_t enmCall : 16;
1314	} ThreadedCall;
1315
1316	struct
1317	{
1318	/* kIemTbDbgEntryType_Label. */
1319	uint32_t uType : 4;
1320	uint32_t uUnused : 4;
1321	/** The label type (IEMNATIVELABELTYPE). */
1322	uint32_t enmLabel : 8;
1323	/** The label data. */
1324	uint32_t uData : 16;
1325	} Label;
1326
1327	struct
1328	{
1329	/* kIemTbDbgEntryType_GuestRegShadowing. */
1330	uint32_t uType : 4;
1331	uint32_t uUnused : 4;
1332	/** The guest register being shadowed (IEMNATIVEGSTREG). */
1333	uint32_t idxGstReg : 8;
1334	/** The host new register number, UINT8_MAX if dropped. */
1335	uint32_t idxHstReg : 8;
1336	/** The previous host register number, UINT8_MAX if new. */
1337	uint32_t idxHstRegPrev : 8;
1338	} GuestRegShadowing;
1339
1340	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1341	struct
1342	{
1343	/* kIemTbDbgEntryType_GuestSimdRegShadowing. */
1344	uint32_t uType : 4;
1345	uint32_t uUnused : 4;
1346	/** The guest register being shadowed (IEMNATIVEGSTSIMDREG). */
1347	uint32_t idxGstSimdReg : 8;
1348	/** The host new register number, UINT8_MAX if dropped. */
1349	uint32_t idxHstSimdReg : 8;
1350	/** The previous host register number, UINT8_MAX if new. */
1351	uint32_t idxHstSimdRegPrev : 8;
1352	} GuestSimdRegShadowing;
1353	#endif
1354
1355	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1356	struct
1357	{
1358	/* kIemTbDbgEntryType_DelayedPcUpdate. */
1359	uint32_t uType : 4;
1360	/** Number of instructions skipped. */
1361	uint32_t cInstrSkipped : 8;
1362	/* The instruction offset added to the program counter. */
1363	int32_t offPc : 20;
1364	} DelayedPcUpdate;
1365	#endif
1366
1367	#if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1368	struct
1369	{
1370	/* kIemTbDbgEntryType_GuestRegDirty. */
1371	uint32_t uType : 4;
1372	uint32_t uUnused : 11;
1373	/** Flag whether this is about a SIMD (true) or general (false) register. */
1374	uint32_t fSimdReg : 1;
1375	/** The guest register index being marked as dirty. */
1376	uint32_t idxGstReg : 8;
1377	/** The host register number this register is shadowed in .*/
1378	uint32_t idxHstReg : 8;
1379	} GuestRegDirty;
1380
1381	struct
1382	{
1383	/* kIemTbDbgEntryType_GuestRegWriteback. */
1384	uint32_t uType : 4;
1385	/** Flag whether this is about a SIMD (true) or general (false) register flush. */
1386	uint32_t fSimdReg : 1;
1387	/** The mask shift. */
1388	uint32_t cShift : 2;
1389	/** The guest register mask being written back. */
1390	uint32_t fGstReg : 25;
1391	} GuestRegWriteback;
1392	#endif
1393
1394	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1395	struct
1396	{
1397	/* kIemTbDbgEntryType_PostponedEFlagsCalc. */
1398	uint32_t uType : 4;
1399	/** The EFLAGS operation (IEMNATIVE_POSTPONED_EFL_OP_T). */
1400	uint32_t enmOp : 4;
1401	/** The mask shift. */
1402	uint32_t cOpBits : 8;
1403	/** The emit instance number (0-based). */
1404	uint32_t idxEmit : 8;
1405	/** Unused. */
1406	uint32_t uUnused : 8;
1407	} PostponedEflCalc;
1408	#endif
1409	} IEMTBDBGENTRY;
1410	AssertCompileSize(IEMTBDBGENTRY, sizeof(uint32_t));
1411	/** Pointer to a debug info entry. */
1412	typedef IEMTBDBGENTRY *PIEMTBDBGENTRY;
1413	/** Pointer to a const debug info entry. */
1414	typedef IEMTBDBGENTRY const *PCIEMTBDBGENTRY;
1415
1416	/**
1417	* Translation block debug info.
1418	*/
1419	typedef struct IEMTBDBG
1420	{
1421	/** This is the flat PC corresponding to IEMTB::GCPhysPc. */
1422	RTGCPTR FlatPc;
1423	/** Number of entries in aEntries. */
1424	uint32_t cEntries;
1425	/** The offset of the last kIemTbDbgEntryType_NativeOffset record. */
1426	uint32_t offNativeLast;
1427	/** Debug info entries. */
1428	RT_FLEXIBLE_ARRAY_EXTENSION
1429	IEMTBDBGENTRY aEntries[RT_FLEXIBLE_ARRAY];
1430	} IEMTBDBG;
1431	/** Pointer to TB debug info. */
1432	typedef IEMTBDBG *PIEMTBDBG;
1433	/** Pointer to const TB debug info. */
1434	typedef IEMTBDBG const *PCIEMTBDBG;
1435
1436	/**
1437	* Guest registers that can be shadowed in GPRs.
1438	*
1439	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
1440	* must be placed last, as the liveness state tracks it as 7 subcomponents and
1441	* we don't want to waste space here.
1442	*
1443	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
1444	* friends as well as IEMAllN8veLiveness.cpp.
1445	*/
1446	typedef enum IEMNATIVEGSTREG : uint8_t
1447	{
1448	kIemNativeGstReg_GprFirst = 0,
1449	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
1450	kIemNativeGstReg_Cr0,
1451	kIemNativeGstReg_Cr4,
1452	kIemNativeGstReg_FpuFcw,
1453	kIemNativeGstReg_FpuFsw,
1454	kIemNativeGstReg_SegBaseFirst,
1455	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
1456	kIemNativeGstReg_SegAttribFirst,
1457	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
1458	kIemNativeGstReg_SegLimitFirst,
1459	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
1460	kIemNativeGstReg_SegSelFirst,
1461	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
1462	kIemNativeGstReg_Xcr0,
1463	kIemNativeGstReg_MxCsr,
1464	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags. /
1465	/* 6 entry gap for liveness EFlags subdivisions. */
1466	kIemNativeGstReg_Pc = kIemNativeGstReg_EFlags + 7,
1467	kIemNativeGstReg_End
1468	} IEMNATIVEGSTREG;
1469	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
1470	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
1471	AssertCompile(RT_BIT_64(kIemNativeGstReg_Pc) - UINT64_C(1) == IEMLIVENESSBIT_MASK);
1472
1473	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
1474	* @{ */
1475	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
1476	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
1477	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
1478	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
1479	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
1480	/** @} */
1481
1482	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1483
1484	/**
1485	* Guest registers that can be shadowed in host SIMD registers.
1486	*
1487	* @todo r=aeichner Liveness tracking
1488	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
1489	*/
1490	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
1491	{
1492	kIemNativeGstSimdReg_SimdRegFirst = 0,
1493	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
1494	kIemNativeGstSimdReg_End
1495	} IEMNATIVEGSTSIMDREG;
1496
1497	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
1498	* @{ */
1499	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
1500	/** @} */
1501
1502	/**
1503	* The Load/store size for a SIMD guest register.
1504	*/
1505	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
1506	{
1507	/** Invalid size. */
1508	kIemNativeGstSimdRegLdStSz_Invalid = 0,
1509	/** Loads the low 128-bit of a guest SIMD register. */
1510	kIemNativeGstSimdRegLdStSz_Low128,
1511	/** Loads the high 128-bit of a guest SIMD register. */
1512	kIemNativeGstSimdRegLdStSz_High128,
1513	/** Loads the whole 256-bits of a guest SIMD register. */
1514	kIemNativeGstSimdRegLdStSz_256,
1515	/** End value. */
1516	kIemNativeGstSimdRegLdStSz_End
1517	} IEMNATIVEGSTSIMDREGLDSTSZ;
1518
1519	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
1520
1521	/**
1522	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
1523	*/
1524	typedef enum IEMNATIVEGSTREGUSE
1525	{
1526	/** The usage is read-only, the register holding the guest register
1527	* shadow copy will not be modified by the caller. */
1528	kIemNativeGstRegUse_ReadOnly = 0,
1529	/** The caller will update the guest register (think: PC += cbInstr).
1530	* The guest shadow copy will follow the returned register. */
1531	kIemNativeGstRegUse_ForUpdate,
1532	/** The call will put an entirely new value in the guest register, so
1533	* if new register is allocate it will be returned uninitialized. */
1534	kIemNativeGstRegUse_ForFullWrite,
1535	/** The caller will use the guest register value as input in a calculation
1536	* and the host register will be modified.
1537	* This means that the returned host register will not be marked as a shadow
1538	* copy of the guest register. */
1539	kIemNativeGstRegUse_Calculation
1540	} IEMNATIVEGSTREGUSE;
1541
1542	/**
1543	* Guest registers (classes) that can be referenced.
1544	*/
1545	typedef enum IEMNATIVEGSTREGREF : uint8_t
1546	{
1547	kIemNativeGstRegRef_Invalid = 0,
1548	kIemNativeGstRegRef_Gpr,
1549	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
1550	kIemNativeGstRegRef_EFlags,
1551	kIemNativeGstRegRef_MxCsr,
1552	kIemNativeGstRegRef_FpuReg,
1553	kIemNativeGstRegRef_MReg,
1554	kIemNativeGstRegRef_XReg,
1555	kIemNativeGstRegRef_X87,
1556	kIemNativeGstRegRef_XState,
1557	//kIemNativeGstRegRef_YReg, - doesn't work.
1558	kIemNativeGstRegRef_End
1559	} IEMNATIVEGSTREGREF;
1560
1561
1562	/** Variable kinds. */
1563	typedef enum IEMNATIVEVARKIND : uint8_t
1564	{
1565	/** Customary invalid zero value. */
1566	kIemNativeVarKind_Invalid = 0,
1567	/** This is either in a register or on the stack. */
1568	kIemNativeVarKind_Stack,
1569	/** Immediate value - loaded into register when needed, or can live on the
1570	* stack if referenced (in theory). */
1571	kIemNativeVarKind_Immediate,
1572	/** Variable reference - loaded into register when needed, never stack. */
1573	kIemNativeVarKind_VarRef,
1574	/** Guest register reference - loaded into register when needed, never stack. */
1575	kIemNativeVarKind_GstRegRef,
1576	/** End of valid values. */
1577	kIemNativeVarKind_End
1578	} IEMNATIVEVARKIND;
1579
1580
1581	/** Variable or argument. */
1582	typedef struct IEMNATIVEVAR
1583	{
1584	/** The kind of variable. */
1585	IEMNATIVEVARKIND enmKind;
1586	/** The variable size in bytes. */
1587	uint8_t cbVar;
1588	/** The first stack slot (uint64_t), except for immediate and references
1589	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1590	* has a stack slot it has been initialized and has a value. Unused variables
1591	* has neither a stack slot nor a host register assignment. */
1592	uint8_t idxStackSlot;
1593	/** The host register allocated for the variable, UINT8_MAX if not. */
1594	uint8_t idxReg;
1595	/** The argument number if argument, UINT8_MAX if regular variable. */
1596	uint8_t uArgNo;
1597	/** If referenced, the index (unpacked) of the variable referencing this one,
1598	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1599	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1600	uint8_t idxReferrerVar;
1601	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1602	* @todo not sure what this really is for... */
1603	IEMNATIVEGSTREG enmGstReg;
1604	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1605	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1606	* only valid when idxReg is not UINT8_MAX. */
1607	bool fSimdReg : 1;
1608	/** Set if the registered is currently used exclusively, false if the
1609	* variable is idle and the register can be grabbed. */
1610	bool fRegAcquired : 1;
1611	#else
1612	/** Set if the registered is currently used exclusively, false if the
1613	* variable is idle and the register can be grabbed. */
1614	bool fRegAcquired;
1615	#endif
1616
1617	union
1618	{
1619	/** kIemNativeVarKind_Immediate: The immediate value. */
1620	uint64_t uValue;
1621	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1622	uint8_t idxRefVar;
1623	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1624	struct
1625	{
1626	/** The class of register. */
1627	IEMNATIVEGSTREGREF enmClass;
1628	/** Index within the class. */
1629	uint8_t idx;
1630	} GstRegRef;
1631	} u;
1632	} IEMNATIVEVAR;
1633	/** Pointer to a variable or argument. */
1634	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1635	/** Pointer to a const variable or argument. */
1636	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1637
1638	/** What is being kept in a host register. */
1639	typedef enum IEMNATIVEWHAT : uint8_t
1640	{
1641	/** The traditional invalid zero value. */
1642	kIemNativeWhat_Invalid = 0,
1643	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1644	kIemNativeWhat_Var,
1645	/** Temporary register, this is typically freed when a MC completes. */
1646	kIemNativeWhat_Tmp,
1647	/** Call argument w/o a variable mapping. This is free (via
1648	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1649	kIemNativeWhat_Arg,
1650	/** Return status code.
1651	* @todo not sure if we need this... */
1652	kIemNativeWhat_rc,
1653	/** The fixed pVCpu (PVMCPUCC) register.
1654	* @todo consider offsetting this on amd64 to use negative offsets to access
1655	* more members using 8-byte disp. */
1656	kIemNativeWhat_pVCpuFixed,
1657	/** The fixed pCtx (PCPUMCTX) register.
1658	* @todo consider offsetting this on amd64 to use negative offsets to access
1659	* more members using 8-byte disp. */
1660	kIemNativeWhat_pCtxFixed,
1661	/** Fixed temporary register. */
1662	kIemNativeWhat_FixedTmp,
1663	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1664	/** Shadow RIP for the delayed RIP updating debugging. */
1665	kIemNativeWhat_PcShadow,
1666	#endif
1667	/** Register reserved by the CPU or OS architecture. */
1668	kIemNativeWhat_FixedReserved,
1669	/** End of valid values. */
1670	kIemNativeWhat_End
1671	} IEMNATIVEWHAT;
1672
1673	/**
1674	* Host general register entry.
1675	*
1676	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1677	*
1678	* @todo Track immediate values in host registers similarlly to how we track the
1679	* guest register shadow copies. For it to be real helpful, though,
1680	* we probably need to know which will be reused and put them into
1681	* non-volatile registers, otherwise it's going to be more or less
1682	* restricted to an instruction or two.
1683	*/
1684	typedef struct IEMNATIVEHSTREG
1685	{
1686	/** Set of guest registers this one shadows.
1687	*
1688	* Using a bitmap here so we can designate the same host register as a copy
1689	* for more than one guest register. This is expected to be useful in
1690	* situations where one value is copied to several registers in a sequence.
1691	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1692	* sequence we'd want to let this register follow to be a copy of and there
1693	* will always be places where we'd be picking the wrong one.
1694	*/
1695	uint64_t fGstRegShadows;
1696	/** What is being kept in this register. */
1697	IEMNATIVEWHAT enmWhat;
1698	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1699	uint8_t idxVar;
1700	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1701	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1702	* that scope. */
1703	uint8_t idxStackSlot;
1704	/** Alignment padding. */
1705	uint8_t abAlign[5];
1706	} IEMNATIVEHSTREG;
1707
1708
1709	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1710	/**
1711	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1712	* halves, on architectures where there is no 256-bit register available this entry will track
1713	* two adjacent 128-bit host registers.
1714	*
1715	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1716	*/
1717	typedef struct IEMNATIVEHSTSIMDREG
1718	{
1719	/** Set of guest registers this one shadows.
1720	*
1721	* Using a bitmap here so we can designate the same host register as a copy
1722	* for more than one guest register. This is expected to be useful in
1723	* situations where one value is copied to several registers in a sequence.
1724	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1725	* sequence we'd want to let this register follow to be a copy of and there
1726	* will always be places where we'd be picking the wrong one.
1727	*/
1728	uint64_t fGstRegShadows;
1729	/** What is being kept in this register. */
1730	IEMNATIVEWHAT enmWhat;
1731	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1732	uint8_t idxVar;
1733	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1734	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1735	/** Alignment padding. */
1736	uint8_t abAlign[5];
1737	} IEMNATIVEHSTSIMDREG;
1738	#endif
1739
1740
1741	/**
1742	* Core state for the native recompiler, that is, things that needs careful
1743	* handling when dealing with branches.
1744	*/
1745	typedef struct IEMNATIVECORESTATE
1746	{
1747	/** Allocation bitmap for aHstRegs. */
1748	uint32_t bmHstRegs;
1749
1750	/** Bitmap marking which host register contains guest register shadow copies.
1751	* This is used during register allocation to try preserve copies. */
1752	uint32_t bmHstRegsWithGstShadow;
1753	/** Bitmap marking valid entries in aidxGstRegShadows. */
1754	uint64_t bmGstRegShadows;
1755	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1756	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1757	uint64_t bmGstRegShadowDirty;
1758	#endif
1759
1760	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1761	/** The current instruction offset in bytes from when the guest program counter
1762	* was updated last. Used for delaying the write to the guest context program counter
1763	* as long as possible. */
1764	int64_t offPc;
1765	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1766	/** Set after we've loaded PC into uPcUpdatingDebug at the first update. */
1767	bool fDebugPcInitialized;
1768	# endif
1769	#endif
1770
1771	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1772	/** Allocation bitmap for aHstSimdRegs. */
1773	uint32_t bmHstSimdRegs;
1774
1775	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1776	* This is used during register allocation to try preserve copies. */
1777	uint32_t bmHstSimdRegsWithGstShadow;
1778	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1779	uint64_t bmGstSimdRegShadows;
1780	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1781	uint64_t bmGstSimdRegShadowDirtyLo128;
1782	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1783	uint64_t bmGstSimdRegShadowDirtyHi128;
1784	#endif
1785
1786	union
1787	{
1788	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1789	uint8_t aidxArgVars[8];
1790	/** For more efficient resetting. */
1791	uint64_t u64ArgVars;
1792	};
1793
1794	/** Allocation bitmap for the stack. */
1795	uint32_t bmStack;
1796	/** Allocation bitmap for aVars. */
1797	uint32_t bmVars;
1798
1799	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1800	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1801	* (A shadow copy of a guest register can only be held in a one host register,
1802	* there are no duplicate copies or ambiguities like that). */
1803	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1804	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1805	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1806	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1807	* (A shadow copy of a guest register can only be held in a one host register,
1808	* there are no duplicate copies or ambiguities like that). */
1809	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1810	#endif
1811
1812	/** Host register allocation tracking. */
1813	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1814	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1815	/** Host SIMD register allocation tracking. */
1816	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1817	#endif
1818
1819	/** Variables and arguments. */
1820	IEMNATIVEVAR aVars[9];
1821	} IEMNATIVECORESTATE;
1822	/** Pointer to core state. */
1823	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1824	/** Pointer to const core state. */
1825	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1826
1827	/** @def IEMNATIVE_VAR_IDX_UNPACK
1828	* @returns Index into IEMNATIVECORESTATE::aVars.
1829	* @param a_idxVar Variable index w/ magic (in strict builds).
1830	*/
1831	/** @def IEMNATIVE_VAR_IDX_PACK
1832	* @returns Variable index w/ magic (in strict builds).
1833	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1834	*/
1835	#ifdef VBOX_STRICT
1836	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1837	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1838	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1839	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1840	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1841	#else
1842	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1843	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1844	#endif
1845
1846
1847	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1848	/** Clear the dirty state of the given guest SIMD register. */
1849	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1850	do { \
1851	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1852	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1853	} while (0)
1854
1855	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1856	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1857	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1858	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1859	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1860	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1861	/** Returns whether the given guest SIMD register is dirty. */
1862	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1863	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1864
1865	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1866	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1867	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1868	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1869	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1870	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1871
1872	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1873	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1874	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1875	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1876	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1877	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1878	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1879	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1880	# ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1881	/** Flag indicating that the guest MXCSR was synced to the host floating point control register. */
1882	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED RT_BIT_32(4)
1883	/** Flag indicating whether the host floating point control register was saved before overwriting it. */
1884	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SAVED RT_BIT_32(5)
1885	# endif
1886	#endif
1887
1888
1889	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1890	typedef enum IEMNATIVE_POSTPONED_EFL_OP_T : uint8_t
1891	{
1892	kIemNativePostponedEflOp_Invalid = 0,
1893	/** Logical operation.
1894	* Operands: result register.
1895	* @note This clears OF, CF and (undefined) AF, thus no need for inputs. */
1896	kIemNativePostponedEflOp_Logical,
1897	kIemNativePostponedEflOp_End
1898	} IEMNATIVE_POSTPONED_EFL_OP_T;
1899	#endif /* IEMNATIVE_WITH_EFLAGS_POSTPONING */
1900
1901	/**
1902	* Conditional stack entry.
1903	*/
1904	typedef struct IEMNATIVECOND
1905	{
1906	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1907	bool fInElse;
1908	union
1909	{
1910	RT_GCC_EXTENSION struct
1911	{
1912	/** Set if the if-block unconditionally exited the TB. */
1913	bool fIfExitTb;
1914	/** Set if the else-block unconditionally exited the TB. */
1915	bool fElseExitTb;
1916	};
1917	/** Indexed by fInElse. */
1918	bool afExitTb[2];
1919	};
1920	bool afPadding[5];
1921	/** The label for the IEM_MC_ELSE. */
1922	uint32_t idxLabelElse;
1923	/** The label for the IEM_MC_ENDIF. */
1924	uint32_t idxLabelEndIf;
1925	/** The initial state snapshot as the if-block starts executing. */
1926	IEMNATIVECORESTATE InitialState;
1927	/** The state snapshot at the end of the if-block. */
1928	IEMNATIVECORESTATE IfFinalState;
1929	} IEMNATIVECOND;
1930	/** Pointer to a condition stack entry. */
1931	typedef IEMNATIVECOND *PIEMNATIVECOND;
1932
1933
1934	/**
1935	* Native recompiler state.
1936	*/
1937	typedef struct IEMRECOMPILERSTATE
1938	{
1939	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1940	* IEMNATIVEINSTR units. */
1941	uint32_t cInstrBufAlloc;
1942	#ifdef VBOX_STRICT
1943	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1944	uint32_t offInstrBufChecked;
1945	#else
1946	uint32_t uPadding1; /* We don't keep track of the size here... */
1947	#endif
1948	/** Fixed temporary code buffer for native recompilation. */
1949	PIEMNATIVEINSTR pInstrBuf;
1950
1951	/** Bitmaps with the label types used. */
1952	uint64_t bmLabelTypes;
1953	/** Actual number of labels in paLabels. */
1954	uint32_t cLabels;
1955	/** Max number of entries allowed in paLabels before reallocating it. */
1956	uint32_t cLabelsAlloc;
1957	/** Labels defined while recompiling (referenced by fixups). */
1958	PIEMNATIVELABEL paLabels;
1959	/** Array with indexes of unique labels (uData always 0). */
1960	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1961
1962	/** Actual number of fixups paFixups. */
1963	uint32_t cFixups;
1964	/** Max number of entries allowed in paFixups before reallocating it. */
1965	uint32_t cFixupsAlloc;
1966	/** Buffer used by the recompiler for recording fixups when generating code. */
1967	PIEMNATIVEFIXUP paFixups;
1968
1969	/** Actual number of fixups in paTbExitFixups. */
1970	uint32_t cTbExitFixups;
1971	/** Max number of entries allowed in paTbExitFixups before reallocating it. */
1972	uint32_t cTbExitFixupsAlloc;
1973	/** Buffer used by the recompiler for recording fixups when generating code. */
1974	PIEMNATIVEEXITFIXUP paTbExitFixups;
1975
1976	#if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) \|\| defined(VBOX_WITH_STATISTICS)
1977	/** Statistics: The idxInstr+1 value at the last PC update. */
1978	uint8_t idxInstrPlusOneOfLastPcUpdate;
1979	#endif
1980
1981	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1982	/** Number of debug info entries allocated for pDbgInfo. */
1983	uint32_t cDbgInfoAlloc;
1984	/** Debug info. */
1985	PIEMTBDBG pDbgInfo;
1986	#endif
1987
1988	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1989	/** The current call index (liveness array and threaded calls in TB). */
1990	uint32_t idxCurCall;
1991	/** Number of liveness entries allocated. */
1992	uint32_t cLivenessEntriesAlloc;
1993	/** Liveness entries for all the calls in the TB begin recompiled.
1994	* The entry for idxCurCall contains the info for what the next call will
1995	* require wrt registers. (Which means the last entry is the initial liveness
1996	* state.) */
1997	PIEMLIVENESSENTRY paLivenessEntries;
1998	#endif
1999
2000	/** The translation block being recompiled. */
2001	PCIEMTB pTbOrg;
2002	/** The VMCPU structure of the EMT. */
2003	PVMCPUCC pVCpu;
2004
2005	/** Condition sequence number (for generating unique labels). */
2006	uint16_t uCondSeqNo;
2007	/** Check IRQ sequence number (for generating unique labels). */
2008	uint16_t uCheckIrqSeqNo;
2009	/** TLB load sequence number (for generating unique labels). */
2010	uint16_t uTlbSeqNo;
2011	/** The current condition stack depth (aCondStack). */
2012	uint8_t cCondDepth;
2013
2014	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
2015	uint8_t cArgsX;
2016	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
2017	uint32_t fCImpl;
2018	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
2019	uint32_t fMc;
2020	/** The expected IEMCPU::fExec value for the current call/instruction. */
2021	uint32_t fExec;
2022	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2023	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
2024	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
2025	*
2026	* This is an optimization because these control registers can only be changed from
2027	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
2028	* consisting of multiple SIMD instructions.
2029	*/
2030	uint32_t fSimdRaiseXcptChecksEmitted;
2031	#endif
2032	/** The call number of the last CheckIrq, UINT32_MAX if not seen. */
2033	uint32_t idxLastCheckIrqCallNo;
2034	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
2035	uint32_t fSkippingEFlags;
2036	#endif
2037	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
2038	struct
2039	{
2040	/** EFLAGS status bits that we're currently postponing the calculcation of. */
2041	uint32_t fEFlags;
2042	/** The postponed EFLAGS status bits calculation operation. */
2043	IEMNATIVE_POSTPONED_EFL_OP_T enmOp;
2044	/** The bit-width of the postponed EFLAGS calculation. */
2045	uint8_t cOpBits;
2046	/** Host register holding result or first source for the delayed operation,
2047	* UINT8_MAX if not in use. */
2048	uint8_t idxReg1;
2049	/** Host register holding second source for the delayed operation,
2050	* UINT8_MAX if not in use. */
2051	uint8_t idxReg2;
2052	# if defined(VBOX_WITH_STATISTICS) \|\| defined(IEMNATIVE_WITH_TB_DEBUG_INFO)
2053	/** Number of times the delayed calculation was emitted. */
2054	uint8_t cEmits;
2055	# endif
2056	} PostponedEfl;
2057	#endif
2058
2059	/** Core state requiring care with branches. */
2060	IEMNATIVECORESTATE Core;
2061
2062	/** The condition nesting stack. */
2063	IEMNATIVECOND aCondStack[2];
2064
2065	#ifndef IEM_WITH_THROW_CATCH
2066	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
2067	* for recompilation error handling. */
2068	jmp_buf JmpBuf;
2069	#endif
2070	} IEMRECOMPILERSTATE;
2071	/** Pointer to a native recompiler state. */
2072	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
2073
2074
2075	/** @def IEMNATIVE_TRY_SETJMP
2076	* Wrapper around setjmp / try, hiding all the ugly differences.
2077	*
2078	* @note Use with extreme care as this is a fragile macro.
2079	* @param a_pReNative The native recompile state.
2080	* @param a_rcTarget The variable that should receive the status code in case
2081	* of a longjmp/throw.
2082	*/
2083	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
2084	* Start wrapper for catch / setjmp-else.
2085	*
2086	* This will set up a scope.
2087	*
2088	* @note Use with extreme care as this is a fragile macro.
2089	* @param a_pReNative The native recompile state.
2090	* @param a_rcTarget The variable that should receive the status code in case
2091	* of a longjmp/throw.
2092	*/
2093	/** @def IEMNATIVE_CATCH_LONGJMP_END
2094	* End wrapper for catch / setjmp-else.
2095	*
2096	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
2097	* up the state.
2098	*
2099	* @note Use with extreme care as this is a fragile macro.
2100	* @param a_pReNative The native recompile state.
2101	*/
2102	/** @def IEMNATIVE_DO_LONGJMP
2103	*
2104	* Wrapper around longjmp / throw.
2105	*
2106	* @param a_pReNative The native recompile state.
2107	* @param a_rc The status code jump back with / throw.
2108	*/
2109	#ifdef IEM_WITH_THROW_CATCH
2110	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
2111	a_rcTarget = VINF_SUCCESS; \
2112	try
2113	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
2114	catch (int rcThrown) \
2115	{ \
2116	a_rcTarget = rcThrown
2117	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
2118	} \
2119	((void)0)
2120	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
2121	#else /* !IEM_WITH_THROW_CATCH */
2122	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
2123	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
2124	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
2125	else \
2126	{ \
2127	((void)0)
2128	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
2129	}
2130	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
2131	#endif /* !IEM_WITH_THROW_CATCH */
2132
2133
2134	/**
2135	* Native recompiler worker for a threaded function.
2136	*
2137	* @returns New code buffer offset; throws VBox status code in case of a failure.
2138	* @param pReNative The native recompiler state.
2139	* @param off The current code buffer offset.
2140	* @param pCallEntry The threaded call entry.
2141	*
2142	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
2143	*/
2144	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
2145	/** Pointer to a native recompiler worker for a threaded function. */
2146	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
2147
2148	/** Defines a native recompiler worker for a threaded function.
2149	* @see FNIEMNATIVERECOMPFUNC */
2150	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
2151	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
2152
2153	/** Prototypes a native recompiler function for a threaded function.
2154	* @see FNIEMNATIVERECOMPFUNC */
2155	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
2156
2157
2158	/**
2159	* Native recompiler liveness analysis worker for a threaded function.
2160	*
2161	* @param pCallEntry The threaded call entry.
2162	* @param pIncoming The incoming liveness state entry.
2163	* @param pOutgoing The outgoing liveness state entry.
2164	*/
2165	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
2166	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
2167	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
2168	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
2169
2170	/** Defines a native recompiler liveness analysis worker for a threaded function.
2171	* @see FNIEMNATIVELIVENESSFUNC */
2172	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
2173	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
2174
2175	/** Prototypes a native recompiler liveness analysis function for a threaded function.
2176	* @see FNIEMNATIVELIVENESSFUNC */
2177	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
2178
2179
2180	/** Define a native recompiler helper function, safe to call from the TB code. */
2181	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
2182	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
2183	/** Prototype a native recompiler helper function, safe to call from the TB code. */
2184	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
2185	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
2186	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
2187	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
2188	a_RetType (VBOXCALL *a_Name) a_ArgList
2189
2190
2191	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2192	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
2193	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
2194	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
2195	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2196	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
2197	IEMNATIVEGSTSIMDREG enmGstSimdReg,
2198	uint8_t idxHstSimdReg = UINT8_MAX,
2199	uint8_t idxHstSimdRegPrev = UINT8_MAX);
2200	# endif
2201	# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
2202	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
2203	uint8_t idxGstReg, uint8_t idxHstReg);
2204	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
2205	uint64_t fGstReg);
2206	# endif
2207	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
2208	uint64_t offPc, uint32_t cInstrSkipped);
2209	# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
2210	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddPostponedEFlagsCalc(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2211	IEMNATIVE_POSTPONED_EFL_OP_T enmOp, uint8_t cOpBits,
2212	uint8_t idxInstance);
2213	# endif
2214	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
2215
2216	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
2217	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
2218	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
2219	DECLHIDDEN(uint32_t) iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
2220	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
2221	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
2222	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
2223	DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere,
2224	IEMNATIVELABELTYPE enmExitReason);
2225	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
2226
2227	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff);
2228	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpPreferNonVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff);
2229	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask);
2230	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpExPreferNonVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask);
2231	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm);
2232
2233	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegReadOnly(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2234	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegUpdate(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2235	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegFullWrite(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2236	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegCalculation(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2237	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegReadOnlyNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2238	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegUpdateNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2239	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegFullWriteNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2240	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegCalculationNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2241
2242	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && defined(VBOX_STRICT)
2243	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestEFlagsReadOnly(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2244	uint64_t fRead, uint64_t fWrite = 0, uint64_t fPotentialCall = 0);
2245	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestEFlagsForUpdate(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2246	uint64_t fRead, uint64_t fWrite = 0, uint64_t fPotentialCall = 0);
2247	#endif
2248
2249	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2250	IEMNATIVEGSTREG enmGstReg);
2251	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && defined(VBOX_STRICT)
2252	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestEFlagsIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2253	uint64_t fRead, uint64_t fWrite = 0);
2254	#else
2255	DECL_FORCE_INLINE_THROW(uint8_t)
2256	iemNativeRegAllocTmpForGuestEFlagsIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2257	uint64_t fRead, uint64_t fWrite = 0)
2258	{
2259	RT_NOREF(fRead, fWrite);
2260	return iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(pReNative, poff, kIemNativeGstReg_EFlags);
2261	}
2262	#endif
2263
2264	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
2265	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
2266	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
2267	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
2268	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
2269	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2270	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
2271	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
2272	# endif
2273	#endif
2274	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
2275	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
2276	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
2277	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
2278	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2279	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
2280	# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2281	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
2282	# endif
2283	#endif
2284	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
2285	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
2286	uint32_t fKeepVars = 0);
2287	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
2288	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
2289	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2290	uint32_t fHstRegsActiveShadows);
2291	#ifdef VBOX_STRICT
2292	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
2293	#endif
2294	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
2295	uint64_t fGstSimdShwExcept);
2296	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2297	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
2298	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off);
2299	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheckWithReg(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxPcReg);
2300	# endif
2301	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
2302	#endif
2303	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2304	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
2305	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWriteEx(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2306	PIEMNATIVECORESTATE pCore, IEMNATIVEGSTREG enmGstReg);
2307	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2308	uint64_t fFlushGstReg = UINT64_MAX);
2309	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative,
2310	uint32_t off, uint8_t idxHstReg);
2311	#endif
2312
2313
2314	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2315	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
2316	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
2317	bool fPreferVolatile = true);
2318	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2319	IEMNATIVEGSTSIMDREG enmGstSimdReg,
2320	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
2321	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
2322	bool fNoVolatileRegs = false);
2323	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
2324	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
2325	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2326	IEMNATIVEGSTSIMDREG enmGstSimdReg);
2327	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2328	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
2329	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
2330	#endif
2331
2332	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
2333	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
2334	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
2335	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
2336	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
2337	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
2338	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
2339	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
2340	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
2341	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
2342	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
2343	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
2344	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
2345	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2346	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
2347	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
2348	#endif
2349	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
2350	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
2351	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2352	uint32_t fHstRegsNotToSave);
2353	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2354	uint32_t fHstRegsNotToSave);
2355	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
2356	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
2357
2358	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2359	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
2360	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowRegEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off,
2361	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
2362	#ifdef VBOX_STRICT
2363	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
2364	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
2365	IEMNATIVEGSTREG enmGstReg);
2366	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheckEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf,
2367	uint32_t off, uint8_t idxReg, IEMNATIVEGSTREG enmGstReg);
2368	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2369	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
2370	IEMNATIVEGSTSIMDREG enmGstSimdReg,
2371	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
2372	# endif
2373	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
2374	#endif
2375	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
2376	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
2377	#endif
2378	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
2379	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
2380	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
2381	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
2382	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
2383	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2384	PCIEMTHRDEDCALLENTRY pCallEntry);
2385	IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(iemNativeLivenessFunc_ThreadedCall);
2386	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
2387	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
2388
2389
2390	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
2391	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
2392	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
2393	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
2394	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
2395	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
2396	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
2397	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
2398	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
2399	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
2400	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
2401	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
2402	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
2403
2404	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2405	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2406	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2407	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2408	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2409	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2410	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2411	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2412	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2413	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2414	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2415	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
2416	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
2417	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
2418	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
2419	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
2420	#endif
2421	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
2422	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
2423	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
2424	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
2425	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2426	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
2427	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
2428	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
2429	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
2430	#endif
2431	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2432	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2433	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2434	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2435	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2436	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2437	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2438
2439	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2440	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2441	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2442	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2443	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2444	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2445	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2446	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2447	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2448	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2449	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2450	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
2451	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
2452	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
2453	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
2454	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
2455	#endif
2456	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
2457	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2458	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2459	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2460	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2461	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
2462	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
2463	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
2464	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
2465	#endif
2466	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2467	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2468	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2469	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2470	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2471	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2472	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2473
2474	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2475	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2476	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2477	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2478	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2479	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2480	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2481	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2482	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2483	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2484	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2485	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2486	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2487	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2488	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2489	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2490	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2491	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2492	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2493	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2494	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2495	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2496
2497	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2498	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2499	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2500	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2501	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2502	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2503	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2504	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2505	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2506	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2507	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2508	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2509	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2510	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2511	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2512	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2513	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2514	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2515	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2516	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2517	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2518	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2519
2520	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2521	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2522	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2523	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2524
2525
2526	/**
2527	* Info about shadowed guest register values.
2528	* @see IEMNATIVEGSTREG
2529	*/
2530	typedef struct IEMANTIVEGSTREGINFO
2531	{
2532	/** Offset in VMCPU. */
2533	uint32_t off;
2534	/** The field size. */
2535	uint8_t cb;
2536	/** Name (for logging). */
2537	const char *pszName;
2538	} IEMANTIVEGSTREGINFO;
2539	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
2540	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
2541	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
2542	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
2543	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
2544
2545
2546
2547	/**
2548	* Ensures that there is sufficient space in the instruction output buffer.
2549	*
2550	* This will reallocate the buffer if needed and allowed.
2551	*
2552	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
2553	* allocation size.
2554	*
2555	* @returns Pointer to the instruction output buffer on success; throws VBox
2556	* status code on failure, so no need to check it.
2557	* @param pReNative The native recompile state.
2558	* @param off Current instruction offset. Works safely for UINT32_MAX
2559	* as well.
2560	* @param cInstrReq Number of instruction about to be added. It's okay to
2561	* overestimate this a bit.
2562	*/
2563	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
2564	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
2565	{
2566	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
2567	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
2568	{
2569	#ifdef VBOX_STRICT
2570	pReNative->offInstrBufChecked = offChecked;
2571	#endif
2572	return pReNative->pInstrBuf;
2573	}
2574	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
2575	}
2576
2577	/**
2578	* Checks that we didn't exceed the space requested in the last
2579	* iemNativeInstrBufEnsure() call.
2580	*/
2581	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
2582	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
2583	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
2584
2585	/**
2586	* Checks that a variable index is valid.
2587	*/
2588	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2589	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2590	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2591	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2592	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
2593	("%s=%#x\n", #a_idxVar, a_idxVar))
2594	#else
2595	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2596	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2597	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
2598	#endif
2599
2600	/**
2601	* Checks that a variable index is valid and that the variable is assigned the
2602	* correct argument number.
2603	* This also adds a RT_NOREF of a_idxVar.
2604	*/
2605	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2606	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2607	RT_NOREF_PV(a_idxVar); \
2608	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2609	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2610	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
2611	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
2612	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2613	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
2614	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
2615	a_uArgNo)); \
2616	} while (0)
2617	#else
2618	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2619	RT_NOREF_PV(a_idxVar); \
2620	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2621	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
2622	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
2623	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2624	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
2625	} while (0)
2626	#endif
2627
2628
2629	/**
2630	* Checks that a variable has the expected size.
2631	*/
2632	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2633	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2634	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2635	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar, (a_cbVar)))
2636
2637
2638	/**
2639	* Calculates the stack address of a variable as a [r]BP displacement value.
2640	*/
2641	DECL_FORCE_INLINE(int32_t)
2642	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2643	{
2644	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2645	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2646	}
2647
2648
2649	/**
2650	* Releases the variable's register.
2651	*
2652	* The register must have been previously acquired calling
2653	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2654	* iemNativeVarRegisterSetAndAcquire().
2655	*/
2656	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2657	{
2658	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2659	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2660	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2661	}
2662
2663
2664	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2665	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2666	{
2667	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2668	iemNativeVarRegisterRelease(pReNative, idxVar);
2669	}
2670	#endif
2671
2672
2673	/**
2674	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2675	*
2676	* @returns The flush mask.
2677	* @param fCImpl The IEM_CIMPL_F_XXX flags.
2678	* @param fGstShwFlush The starting flush mask.
2679	*/
2680	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2681	{
2682	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2683	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2684	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2685	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2686	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2687	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2688	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2689	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2690	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2691	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2692	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2693	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
2694	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
2695	return fGstShwFlush;
2696	}
2697
2698
2699	/** Number of hidden arguments for CIMPL calls.
2700	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2701	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2702	# define IEM_CIMPL_HIDDEN_ARGS 3
2703	#else
2704	# define IEM_CIMPL_HIDDEN_ARGS 2
2705	#endif
2706
2707
2708	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2709	/** Number of hidden arguments for SSE_AIMPL calls. */
2710	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2711	/** Number of hidden arguments for AVX_AIMPL calls. */
2712	# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2713	#endif
2714
2715
2716	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2717
2718	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2719	/**
2720	* Helper for iemNativeLivenessGetStateByGstReg.
2721	*
2722	* @returns IEMLIVENESS_STATE_XXX
2723	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2724	* ORed together.
2725	*/
2726	DECL_FORCE_INLINE(uint32_t)
2727	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2728	{
2729	/* INPUT trumps anything else. */
2730	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2731	return IEMLIVENESS_STATE_INPUT;
2732
2733	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2734	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2735	{
2736	/* If not all sub-fields are clobbered they must be considered INPUT. */
2737	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2738	return IEMLIVENESS_STATE_INPUT;
2739	return IEMLIVENESS_STATE_CLOBBERED;
2740	}
2741
2742	/* XCPT_OR_CALL trumps UNUSED. */
2743	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2744	return IEMLIVENESS_STATE_XCPT_OR_CALL;
2745
2746	return IEMLIVENESS_STATE_UNUSED;
2747	}
2748	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2749
2750
2751	DECL_FORCE_INLINE(uint32_t)
2752	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2753	{
2754	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2755	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2756	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2757	# else
2758	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2759	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2760	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2761	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 3) & 8);
2762	# endif
2763	}
2764
2765
2766	DECL_FORCE_INLINE(uint32_t)
2767	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2768	{
2769	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2770	if (enmGstReg == kIemNativeGstReg_EFlags)
2771	{
2772	/* Merge the eflags states to one. */
2773	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2774	uRet = RT_BIT_32(uRet);
2775	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2776	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2777	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2778	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2779	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2780	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2781	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2782	# else
2783	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2784	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2785	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2786	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2787	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2788	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2789	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2790	# endif
2791	}
2792	return uRet;
2793	}
2794
2795	# ifdef VBOX_STRICT
2796
2797	/** For assertions only - caller checks that idxCurCall isn't zero. */
2798	DECL_FORCE_INLINE(uint32_t)
2799	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2800	{
2801	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2802	}
2803
2804
2805	/** For assertions only - caller checks that idxCurCall isn't zero. */
2806	DECL_FORCE_INLINE(uint32_t)
2807	iemNativeLivenessGetPrevStateByGstRegEx(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2808	{
2809	return iemNativeLivenessGetStateByGstRegEx(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2810	}
2811
2812	# endif /* VBOX_STRICT */
2813	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2814
2815
2816	/**
2817	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2818	*/
2819	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2820	{
2821	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2822	return IEM_CIMPL_HIDDEN_ARGS;
2823	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2824	return 1;
2825	return 0;
2826	}
2827
2828
2829	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2830	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2831	{
2832	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2833
2834	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2835	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2836	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2837	return (uint8_t)idxReg;
2838	}
2839
2840
2841
2842	/*********************************************************************************************************************************
2843	* Register Allocator (GPR) *
2844	*********************************************************************************************************************************/
2845
2846	#ifdef RT_ARCH_ARM64
2847	# include <iprt/armv8.h>
2848	#endif
2849
2850
2851	/**
2852	* Marks host register @a idxHstReg as containing a shadow copy of guest
2853	* register @a enmGstReg.
2854	*
2855	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2856	* host register before calling.
2857	*/
2858	DECL_FORCE_INLINE(void)
2859	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2860	{
2861	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2862	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2863	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2864
2865	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2866	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2867	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2868	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2869	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2870	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2871	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2872	#else
2873	RT_NOREF(off);
2874	#endif
2875	}
2876
2877
2878	/**
2879	* Clear any guest register shadow claims from @a idxHstReg.
2880	*
2881	* The register does not need to be shadowing any guest registers.
2882	*/
2883	DECL_FORCE_INLINE(void)
2884	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2885	{
2886	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2887	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2888	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2889	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2890	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2891	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2892	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2893	#endif
2894
2895	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2896	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2897	if (fGstRegs)
2898	{
2899	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2900	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2901	while (fGstRegs)
2902	{
2903	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2904	fGstRegs &= ~RT_BIT_64(iGstReg);
2905	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2906	}
2907	}
2908	#else
2909	RT_NOREF(off);
2910	#endif
2911
2912	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2913	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2914	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2915	}
2916
2917
2918	/**
2919	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2920	* and global overview flags.
2921	*/
2922	DECL_FORCE_INLINE(void)
2923	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2924	{
2925	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2926	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2927	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2928	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2929	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2930	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2931	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2932	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2933	Assert(!(pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(enmGstReg)));
2934	#endif
2935
2936	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2937	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2938	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2939	#else
2940	RT_NOREF(off);
2941	#endif
2942
2943	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2944	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2945	if (!fGstRegShadowsNew)
2946	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2947	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2948	}
2949
2950
2951	#if 0 /* unused */
2952	/**
2953	* Clear any guest register shadow claim for @a enmGstReg.
2954	*/
2955	DECL_FORCE_INLINE(void)
2956	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2957	{
2958	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2959	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2960	{
2961	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2962	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2963	}
2964	}
2965	#endif
2966
2967
2968	/**
2969	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2970	* as the new shadow of it.
2971	*
2972	* Unlike the other guest reg shadow helpers, this does the logging for you.
2973	* However, it is the liveness state is not asserted here, the caller must do
2974	* that.
2975	*/
2976	DECL_FORCE_INLINE(void)
2977	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2978	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2979	{
2980	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2981	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2982	{
2983	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2984	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2985	if (idxHstRegOld == idxHstRegNew)
2986	return;
2987	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2988	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2989	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2990	}
2991	else
2992	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2993	g_aGstShadowInfo[enmGstReg].pszName));
2994	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2995	}
2996
2997
2998	/**
2999	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
3000	* to @a idxRegTo.
3001	*/
3002	DECL_FORCE_INLINE(void)
3003	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
3004	IEMNATIVEGSTREG enmGstReg, uint32_t off)
3005	{
3006	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
3007	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
3008	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
3009	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
3010	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
3011	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
3012	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
3013	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
3014	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
3015
3016	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
3017	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
3018	if (!fGstRegShadowsFrom)
3019	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
3020	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
3021	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
3022	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
3023	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3024	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3025	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
3026	#else
3027	RT_NOREF(off);
3028	#endif
3029	}
3030
3031
3032	/**
3033	* Flushes any delayed guest register writes.
3034	*
3035	* This must be called prior to calling CImpl functions and any helpers that use
3036	* the guest state (like raising exceptions) and such.
3037	*
3038	* This optimization has not yet been implemented. The first target would be
3039	* RIP updates, since these are the most common ones.
3040	*
3041	* @note This function does not flush any shadowing information for guest
3042	* registers. This needs to be done by the caller if it wishes to do so.
3043	*/
3044	DECL_INLINE_THROW(uint32_t)
3045	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0,
3046	uint64_t fGstSimdShwExcept = 0)
3047	{
3048	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
3049	uint64_t const fWritebackPc = ~fGstShwExcept & RT_BIT_64(kIemNativeGstReg_Pc);
3050	#else
3051	uint64_t const fWritebackPc = 0;
3052	#endif
3053	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
3054	uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
3055	#else
3056	uint64_t const bmGstRegShadowDirty = 0;
3057	#endif
3058	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
3059	uint64_t const bmGstSimdRegShadowDirty = ( pReNative->Core.bmGstSimdRegShadowDirtyLo128
3060	\| pReNative->Core.bmGstSimdRegShadowDirtyHi128)
3061	& ~fGstSimdShwExcept;
3062	#else
3063	uint64_t const bmGstSimdRegShadowDirty = 0;
3064	#endif
3065	if (bmGstRegShadowDirty \| bmGstSimdRegShadowDirty \| fWritebackPc)
3066	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
3067
3068	return off;
3069	}
3070
3071
3072	/**
3073	* Allocates a temporary host general purpose register for keeping a guest
3074	* register value.
3075	*
3076	* Since we may already have a register holding the guest register value,
3077	* code will be emitted to do the loading if that's not the case. Code may also
3078	* be emitted if we have to free up a register to satify the request.
3079	*
3080	* @returns The host register number; throws VBox status code on failure, so no
3081	* need to check the return value.
3082	* @param pReNative The native recompile state.
3083	* @param poff Pointer to the variable with the code buffer
3084	* position. This will be update if we need to move a
3085	* variable from register to stack in order to satisfy
3086	* the request.
3087	* @param enmGstReg The guest register that will is to be updated.
3088	* @param enmIntendedUse How the caller will be using the host register.
3089	* @param fNoVolatileRegs Set if no volatile register allowed, clear if any
3090	* register is okay (default). The ASSUMPTION here is
3091	* that the caller has already flushed all volatile
3092	* registers, so this is only applied if we allocate a
3093	* new register.
3094	* @sa iemNativeRegAllocTmpForGuestEFlags
3095	* iemNativeRegAllocTmpForGuestRegIfAlreadyPresent
3096	* iemNativeRegAllocTmpForGuestRegInt
3097	*/
3098	DECL_FORCE_INLINE_THROW(uint8_t)
3099	iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg,
3100	IEMNATIVEGSTREGUSE const enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
3101	bool const fNoVolatileRegs = false)
3102	{
3103	if (enmIntendedUse == kIemNativeGstRegUse_ReadOnly)
3104	return !fNoVolatileRegs
3105	? iemNativeRegAllocTmpForGuestRegReadOnly(pReNative, poff, enmGstReg)
3106	: iemNativeRegAllocTmpForGuestRegReadOnlyNoVolatile(pReNative, poff, enmGstReg);
3107	if (enmIntendedUse == kIemNativeGstRegUse_ForUpdate)
3108	return !fNoVolatileRegs
3109	? iemNativeRegAllocTmpForGuestRegUpdate(pReNative, poff, enmGstReg)
3110	: iemNativeRegAllocTmpForGuestRegUpdateNoVolatile(pReNative, poff, enmGstReg);
3111	if (enmIntendedUse == kIemNativeGstRegUse_ForFullWrite)
3112	return !fNoVolatileRegs
3113	? iemNativeRegAllocTmpForGuestRegFullWrite(pReNative, poff, enmGstReg)
3114	: iemNativeRegAllocTmpForGuestRegFullWriteNoVolatile(pReNative, poff, enmGstReg);
3115	Assert(enmIntendedUse == kIemNativeGstRegUse_Calculation);
3116	return !fNoVolatileRegs
3117	? iemNativeRegAllocTmpForGuestRegCalculation(pReNative, poff, enmGstReg)
3118	: iemNativeRegAllocTmpForGuestRegCalculationNoVolatile(pReNative, poff, enmGstReg);
3119	}
3120
3121	#if !defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| !defined(VBOX_STRICT)
3122
3123	DECL_FORCE_INLINE_THROW(uint8_t)
3124	iemNativeRegAllocTmpForGuestEFlagsReadOnly(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t fRead,
3125	uint64_t fWrite = 0, uint64_t fPotentialCall = 0)
3126	{
3127	RT_NOREF(fRead, fWrite, fPotentialCall);
3128	return iemNativeRegAllocTmpForGuestRegReadOnly(pReNative, poff, kIemNativeGstReg_EFlags);
3129	}
3130
3131	DECL_FORCE_INLINE_THROW(uint8_t)
3132	iemNativeRegAllocTmpForGuestEFlagsForUpdate(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t fRead,
3133	uint64_t fWrite = 0, uint64_t fPotentialCall = 0)
3134	{
3135	RT_NOREF(fRead, fWrite, fPotentialCall);
3136	return iemNativeRegAllocTmpForGuestRegUpdate(pReNative, poff, kIemNativeGstReg_EFlags);
3137	}
3138
3139	#endif
3140
3141
3142
3143	/*********************************************************************************************************************************
3144	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
3145	*********************************************************************************************************************************/
3146
3147	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
3148
3149	DECL_FORCE_INLINE(uint8_t)
3150	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
3151	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
3152	{
3153	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
3154
3155	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
3156	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
3157	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
3158	return idxSimdReg;
3159	}
3160
3161
3162	/**
3163	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
3164	* SIMD register @a enmGstSimdReg.
3165	*
3166	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
3167	* host register before calling.
3168	*/
3169	DECL_FORCE_INLINE(void)
3170	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
3171	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
3172	{
3173	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
3174	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
3175	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
3176
3177	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
3178	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
3179	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
3180	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
3181	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3182	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3183	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
3184	#else
3185	RT_NOREF(off);
3186	#endif
3187	}
3188
3189
3190	/**
3191	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
3192	* to @a idxSimdRegTo.
3193	*/
3194	DECL_FORCE_INLINE(void)
3195	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
3196	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
3197	{
3198	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
3199	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
3200	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
3201	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
3202	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
3203	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
3204	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
3205	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
3206	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
3207	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
3208	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
3209
3210	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
3211	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
3212	if (!fGstRegShadowsFrom)
3213	{
3214	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
3215	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
3216	}
3217	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
3218	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
3219	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
3220	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3221	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3222	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
3223	#else
3224	RT_NOREF(off);
3225	#endif
3226	}
3227
3228
3229	/**
3230	* Clear any guest register shadow claims from @a idxHstSimdReg.
3231	*
3232	* The register does not need to be shadowing any guest registers.
3233	*/
3234	DECL_FORCE_INLINE(void)
3235	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
3236	{
3237	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
3238	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
3239	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
3240	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
3241	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
3242	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
3243	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
3244
3245	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3246	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
3247	if (fGstRegs)
3248	{
3249	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
3250	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3251	while (fGstRegs)
3252	{
3253	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
3254	fGstRegs &= ~RT_BIT_64(iGstReg);
3255	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
3256	}
3257	}
3258	#else
3259	RT_NOREF(off);
3260	#endif
3261
3262	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
3263	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
3264	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
3265	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
3266	}
3267
3268	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
3269
3270
3271	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
3272	/**
3273	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
3274	*/
3275	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
3276	{
3277	if (pReNative->Core.offPc)
3278	return iemNativeEmitPcWritebackSlow(pReNative, off);
3279	return off;
3280	}
3281	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
3282
3283
3284	/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
3285	* it saves us the trouble of a hidden parameter on MSC/amd64. */
3286	#ifdef RT_ARCH_AMD64
3287	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
3288	#elif defined(RT_ARCH_ARM64)
3289	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
3290	#endif
3291
3292	#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
3293	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeFpCtrlRegRestore, (uint64_t u64RegFpCtrl));
3294	#endif
3295
3296	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
3297
3298	/** @} */
3299
3300	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
3301

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 106319

Download in other formats: