IEMN8veRecompiler.h@ 106401

Last change on this file since 106401 was 106401, checked in by vboxsync, 7 weeks ago
VMM/IEM: Disable control flow guard for the core IEM jump table call loops. bugref:10720
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 159.4 KB

Line
1	/* $Id: IEMN8veRecompiler.h 106401 2024-10-16 20:56:01Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43	* Enables generating internal debug info for better TB disassembly dumping. */
44	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING) \|\| 0
45	# define IEMNATIVE_WITH_TB_DEBUG_INFO
46	#endif
47
48	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49	* Enables liveness analysis. */
50	#if 1 \|\| defined(DOXYGEN_RUNNING)
51	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52	#endif
53
54	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
55	* Enables skipping EFLAGS calculations/updating based on liveness info. */
56	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
57	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
58	#endif
59
60	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
61	* Enables strict consistency checks around EFLAGS skipping.
62	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
63	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
64	# ifdef VBOX_STRICT
65	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
66	# endif
67	#elif defined(DOXYGEN_RUNNING)
68	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
69	#endif
70
71	/** @def IEMNATIVE_WITH_EFLAGS_POSTPONING
72	* Enables delaying EFLAGS calculations/updating to conditional code paths
73	* that are (hopefully) not taken so frequently.
74	*
75	* This can only help with case where there is an conditional
76	* call/exception/tbexit that needs the flag, but in the default code stream the
77	* flag will be clobbered. Useful for TlbMiss scenarios and sequences of memory
78	* based instructions clobbering status flags. */
79	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
80	# if 1 \|\| defined(DOXYGEN_RUNNING)
81	# define IEMNATIVE_WITH_EFLAGS_POSTPONING
82	# endif
83	#endif
84	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
85	# ifndef IEMNATIVE_WITH_EFLAGS_SKIPPING
86	# error "IEMNATIVE_WITH_EFLAGS_POSTPONING requires IEMNATIVE_WITH_EFLAGS_SKIPPING at present"
87	# endif
88	#endif
89
90	/** @def IEMLIVENESS_EXTENDED_LAYOUT
91	* Enables the extended liveness data layout. */
92	#if defined(IEMNATIVE_WITH_EFLAGS_POSTPONING) \|\| defined(DOXYGEN_RUNNING) \|\| 0
93	# define IEMLIVENESS_EXTENDED_LAYOUT
94	#endif
95
96
97	#ifdef VBOX_WITH_STATISTICS
98	/** Always count instructions for now. */
99	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
100	#endif
101
102
103	/** @name Stack Frame Layout
104	*
105	* @{ */
106	/** The size of the area for stack variables and spills and stuff.
107	* @note This limit is duplicated in the python script(s). We add 0x40 for
108	* alignment padding. */
109	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
110	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
111	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
112	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
113
114	#ifdef RT_ARCH_AMD64
115	/** An stack alignment adjustment (between non-volatile register pushes and
116	* the stack variable area, so the latter better aligned). */
117	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
118
119	/** Number of stack arguments slots for calls made from the frame. */
120	# ifdef RT_OS_WINDOWS
121	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
122	# else
123	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
124	# endif
125	/** Number of any shadow arguments (spill area) for calls we make. */
126	# ifdef RT_OS_WINDOWS
127	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
128	# else
129	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
130	# endif
131
132	/** Frame pointer (RBP) relative offset of the last push. */
133	# ifdef RT_OS_WINDOWS
134	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
135	# else
136	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
137	# endif
138	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
139	* address for it). */
140	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
141	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
142	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
143	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
144	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
145	# ifdef RT_OS_WINDOWS
146	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
147	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
148	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
149	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
150	# endif
151
152	# ifdef RT_OS_WINDOWS
153	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
154	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
155	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
156	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
157	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
158	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
159	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
160	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
161	# endif
162
163	#elif RT_ARCH_ARM64
164	/** No alignment padding needed for arm64. */
165	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
166	/** No stack argument slots, got 8 registers for arguments will suffice. */
167	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
168	/** There are no argument spill area. */
169	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
170
171	/** Number of saved registers at the top of our stack frame.
172	* This includes the return address and old frame pointer, so x19 thru x30. */
173	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
174	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
175	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
176
177	/** Frame pointer (BP) relative offset of the last push. */
178	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
179
180	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
181	* address for it). */
182	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
183
184	#else
185	# error "port me"
186	#endif
187	/** @} */
188
189
190	/** @name Fixed Register Allocation(s)
191	* @{ */
192	/** @def IEMNATIVE_REG_FIXED_PVMCPU
193	* The number of the register holding the pVCpu pointer. */
194	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
195	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
196	* @note This not available on AMD64, only ARM64. */
197	/** @def IEMNATIVE_REG_FIXED_TMP0
198	* Dedicated temporary register.
199	* @note This has extremely short lifetime, must be used with great care to make
200	* sure any calling code or code being called is making use of it.
201	* It will definitely not survive a call or anything of that nature.
202	* @todo replace this by a register allocator and content tracker. */
203	/** @def IEMNATIVE_REG_FIXED_MASK
204	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
205	* architecture. */
206	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
207	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
208	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
209	* architecture. */
210	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
211	* Dedicated temporary SIMD register. */
212	#endif
213	#if defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
214	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
215	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
216	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
217	# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
218	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
219	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
220	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
221	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
222	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
223	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
224	# else
225	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
226	# endif
227	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
228	\| RT_BIT_32(ARMV8_A64_REG_LR) \
229	\| RT_BIT_32(ARMV8_A64_REG_BP) \
230	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
231	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
232	\| RT_BIT_32(ARMV8_A64_REG_X18) \
233	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
234	\| IEMNATIVE_REG_FIXED_MASK_ADD)
235
236	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
237	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
238	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
239	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
240	# else
241	/** @note
242	* ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
243	* support emulating 256-bit registers we pair two real registers statically to
244	* one virtual for now, leaving us with only 16 256-bit registers. We always
245	* pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
246	* the register allocator assumes that it will be always free when the lower is
247	* picked.
248	*
249	* Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
250	* touch them in order to avoid having to save and restore them in the
251	* prologue/epilogue.
252	*/
253	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
254	\| RT_BIT_32(ARMV8_A64_REG_Q31) \
255	\| RT_BIT_32(ARMV8_A64_REG_Q30) \
256	\| RT_BIT_32(ARMV8_A64_REG_Q29) \
257	\| RT_BIT_32(ARMV8_A64_REG_Q27) \
258	\| RT_BIT_32(ARMV8_A64_REG_Q25) \
259	\| RT_BIT_32(ARMV8_A64_REG_Q23) \
260	\| RT_BIT_32(ARMV8_A64_REG_Q21) \
261	\| RT_BIT_32(ARMV8_A64_REG_Q19) \
262	\| RT_BIT_32(ARMV8_A64_REG_Q17) \
263	\| RT_BIT_32(ARMV8_A64_REG_Q15) \
264	\| RT_BIT_32(ARMV8_A64_REG_Q13) \
265	\| RT_BIT_32(ARMV8_A64_REG_Q11) \
266	\| RT_BIT_32(ARMV8_A64_REG_Q9) \
267	\| RT_BIT_32(ARMV8_A64_REG_Q7) \
268	\| RT_BIT_32(ARMV8_A64_REG_Q5) \
269	\| RT_BIT_32(ARMV8_A64_REG_Q3) \
270	\| RT_BIT_32(ARMV8_A64_REG_Q1))
271	# endif
272	# endif
273
274	#elif defined(RT_ARCH_AMD64)
275	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
276	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
277	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
278	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
279	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
280	\| RT_BIT_32(X86_GREG_xSP) \
281	\| RT_BIT_32(X86_GREG_xBP) )
282
283	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
284	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
285	# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
286	# ifndef _MSC_VER
287	# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
288	# endif
289	# endif
290	# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
291	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
292	# else
293	/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
294	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
295	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
296	# endif
297	# endif
298
299	#else
300	# error "port me"
301	#endif
302	/** @} */
303
304	/** @name Call related registers.
305	* @{ */
306	/** @def IEMNATIVE_CALL_RET_GREG
307	* The return value register. */
308	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
309	* Number of arguments in registers. */
310	/** @def IEMNATIVE_CALL_ARG0_GREG
311	* The general purpose register carrying argument \#0. */
312	/** @def IEMNATIVE_CALL_ARG1_GREG
313	* The general purpose register carrying argument \#1. */
314	/** @def IEMNATIVE_CALL_ARG2_GREG
315	* The general purpose register carrying argument \#2. */
316	/** @def IEMNATIVE_CALL_ARG3_GREG
317	* The general purpose register carrying argument \#3. */
318	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
319	* Mask of registers the callee will not save and may trash. */
320	#ifdef RT_ARCH_AMD64
321	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
322
323	# ifdef RT_OS_WINDOWS
324	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
325	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
326	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
327	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
328	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
329	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
330	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
331	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
332	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
333	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
334	\| RT_BIT_32(X86_GREG_xCX) \
335	\| RT_BIT_32(X86_GREG_xDX) \
336	\| RT_BIT_32(X86_GREG_x8) \
337	\| RT_BIT_32(X86_GREG_x9) \
338	\| RT_BIT_32(X86_GREG_x10) \
339	\| RT_BIT_32(X86_GREG_x11) )
340	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
341	/* xmm0 - xmm5 are marked as volatile. */
342	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
343	# endif
344
345	# else /* !RT_OS_WINDOWS */
346	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
347	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
348	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
349	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
350	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
351	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
352	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
353	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
354	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
355	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
356	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
357	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
358	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
359	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
360	\| RT_BIT_32(X86_GREG_xCX) \
361	\| RT_BIT_32(X86_GREG_xDX) \
362	\| RT_BIT_32(X86_GREG_xDI) \
363	\| RT_BIT_32(X86_GREG_xSI) \
364	\| RT_BIT_32(X86_GREG_x8) \
365	\| RT_BIT_32(X86_GREG_x9) \
366	\| RT_BIT_32(X86_GREG_x10) \
367	\| RT_BIT_32(X86_GREG_x11) )
368	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
369	/* xmm0 - xmm15 are marked as volatile. */
370	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
371	# endif
372	# endif /* !RT_OS_WINDOWS */
373
374	#elif defined(RT_ARCH_ARM64)
375	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
376	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
377	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
378	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
379	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
380	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
381	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
382	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
383	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
384	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
385	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
386	\| RT_BIT_32(ARMV8_A64_REG_X1) \
387	\| RT_BIT_32(ARMV8_A64_REG_X2) \
388	\| RT_BIT_32(ARMV8_A64_REG_X3) \
389	\| RT_BIT_32(ARMV8_A64_REG_X4) \
390	\| RT_BIT_32(ARMV8_A64_REG_X5) \
391	\| RT_BIT_32(ARMV8_A64_REG_X6) \
392	\| RT_BIT_32(ARMV8_A64_REG_X7) )
393	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
394	\| RT_BIT_32(ARMV8_A64_REG_X1) \
395	\| RT_BIT_32(ARMV8_A64_REG_X2) \
396	\| RT_BIT_32(ARMV8_A64_REG_X3) \
397	\| RT_BIT_32(ARMV8_A64_REG_X4) \
398	\| RT_BIT_32(ARMV8_A64_REG_X5) \
399	\| RT_BIT_32(ARMV8_A64_REG_X6) \
400	\| RT_BIT_32(ARMV8_A64_REG_X7) \
401	\| RT_BIT_32(ARMV8_A64_REG_X8) \
402	\| RT_BIT_32(ARMV8_A64_REG_X9) \
403	\| RT_BIT_32(ARMV8_A64_REG_X10) \
404	\| RT_BIT_32(ARMV8_A64_REG_X11) \
405	\| RT_BIT_32(ARMV8_A64_REG_X12) \
406	\| RT_BIT_32(ARMV8_A64_REG_X13) \
407	\| RT_BIT_32(ARMV8_A64_REG_X14) \
408	\| RT_BIT_32(ARMV8_A64_REG_X15) \
409	\| RT_BIT_32(ARMV8_A64_REG_X16) \
410	\| RT_BIT_32(ARMV8_A64_REG_X17) )
411	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
412	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
413	* so to simplify our life a bit we just mark everything as volatile. */
414	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
415	# endif
416
417	#endif
418
419	/** This is the maximum argument count we'll ever be needing. */
420	#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
421	#ifdef RT_OS_WINDOWS
422	# ifdef VBOXSTRICTRC_STRICT_ENABLED
423	# undef IEMNATIVE_CALL_MAX_ARG_COUNT
424	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
425	# endif
426	#endif
427
428	/** @def IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK
429	* Variant of IEMNATIVE_CALL_VOLATILE_GREG_MASK that excludes
430	* IEMNATIVE_REG_FIXED_TMP0 on hosts that uses it. */
431	#ifdef IEMNATIVE_REG_FIXED_TMP0
432	# ifdef IEMNATIVE_REG_FIXED_TMP1
433	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK ( IEMNATIVE_CALL_VOLATILE_GREG_MASK \
434	& ~( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
435	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1)))
436	# else
437	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK (IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0))
438	# endif
439	#else
440	# define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK IEMNATIVE_CALL_VOLATILE_GREG_MASK
441	#endif
442
443	/** @def IEMNATIVE_CALL_NONVOLATILE_GREG_MASK
444	* The allocatable non-volatile general purpose register set. */
445	#define IEMNATIVE_CALL_NONVOLATILE_GREG_MASK \
446	(~IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK & IEMNATIVE_HST_GREG_MASK)
447	/** @} */
448
449
450	/** @def IEMNATIVE_HST_GREG_COUNT
451	* Number of host general purpose registers we tracker. */
452	/** @def IEMNATIVE_HST_GREG_MASK
453	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
454	* inverted register masks and such to get down to a correct set of regs. */
455	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
456	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
457	* Number of host SIMD registers we track. */
458	/** @def IEMNATIVE_HST_SIMD_REG_MASK
459	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
460	* inverted register masks and such to get down to a correct set of regs. */
461	#endif
462	#ifdef RT_ARCH_AMD64
463	# define IEMNATIVE_HST_GREG_COUNT 16
464	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
465
466	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
467	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
468	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
469	# endif
470
471	#elif defined(RT_ARCH_ARM64)
472	# define IEMNATIVE_HST_GREG_COUNT 32
473	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
474
475	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
476	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
477	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
478	# endif
479
480	#else
481	# error "Port me!"
482	#endif
483
484
485	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
486
487
488	/** Native code generator label types. */
489	typedef enum
490	{
491	kIemNativeLabelType_Invalid = 0,
492	/** @name Exit reasons - Labels w/o data, only once instance per TB.
493	*
494	* The labels requiring register inputs are documented.
495	*
496	* @note Jumps to these requires instructions that are capable of spanning the
497	* max TB length.
498	* @{
499	*/
500	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
501	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
502	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
503	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
504	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
505	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
506	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
507	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
508	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
509	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
510	kIemNativeLabelType_ObsoleteTb, /*< Calls iemNativeHlpObsoleteTb (no inputs). /
511	kIemNativeLabelType_NeedCsLimChecking, /*< Calls iemNativeHlpNeedCsLimChecking (no inputs). /
512	kIemNativeLabelType_CheckBranchMiss, /*< Calls iemNativeHlpCheckBranchMiss (no inputs). /
513	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
514
515	/* Manually defined labels: */
516	/*< Returns with VINF_SUCCESS, no inputs. /
517	kIemNativeLabelType_ReturnSuccess,
518	/** Returns with VINF_IEM_REEXEC_FINISH_WITH_FLAGS, no inputs. */
519	kIemNativeLabelType_ReturnWithFlags,
520	/** Returns with VINF_IEM_REEXEC_BREAK, no inputs. */
521	kIemNativeLabelType_ReturnBreak,
522	/** Returns with VINF_IEM_REEXEC_BREAK_FF, no inputs. */
523	kIemNativeLabelType_ReturnBreakFF,
524	/** The last TB exit label that doesn't have any input registers. */
525	kIemNativeLabelType_LastTbExitWithoutInputs = kIemNativeLabelType_ReturnBreakFF,
526
527	/** Argument registers 1, 2 & 3 are set up. */
528	kIemNativeLabelType_ReturnBreakViaLookup,
529	/** Argument registers 1, 2 & 3 are set up. */
530	kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
531	/** Argument registers 1 & 2 are set up. */
532	kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
533	/** Argument registers 1 & 2 are set up. */
534	kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
535	/** Return register holds the RC and the instruction number is in CL/RCX
536	* on amd64 and the 2rd argument register elsewhere. */
537	kIemNativeLabelType_NonZeroRetOrPassUp,
538
539	/** The last fixup for branches that can span almost the whole TB length.
540	* @note Whether kIemNativeLabelType_Return needs to be one of these is
541	* a bit questionable, since nobody jumps to it except other tail code. */
542	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
543	/** The last fixup for branches that exits the TB. */
544	kIemNativeLabelType_LastTbExit = kIemNativeLabelType_NonZeroRetOrPassUp,
545	/** @} */
546
547	/** Loop-jump target. */
548	kIemNativeLabelType_LoopJumpTarget,
549
550	/*
551	* Labels with data, potentially multiple instances per TB:
552	*
553	* These are localized labels, so no fixed jump type restrictions here.
554	*/
555	kIemNativeLabelType_FirstWithMultipleInstances,
556	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
557	kIemNativeLabelType_Else,
558	kIemNativeLabelType_Endif,
559	kIemNativeLabelType_CheckIrq,
560	kIemNativeLabelType_TlbLookup,
561	kIemNativeLabelType_TlbMiss,
562	kIemNativeLabelType_TlbDone,
563	kIemNativeLabelType_End
564	} IEMNATIVELABELTYPE;
565
566	#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
567	((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
568
569	#define IEMNATIVELABELTYPE_IS_EXIT_WITHOUT_INPUTS(a_enmLabel) \
570	((a_enmLabel) <= kIemNativeLabelType_LastTbExitWithoutInputs && (a_enmLabel) > kIemNativeLabelType_Invalid)
571
572	/**
573	* Get the mask of input registers for an TB exit label.
574	* This will return zero for any non-exit lable.
575	*/
576	#ifdef RT_ARCH_AMD64
577	# define IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmLabel) \
578	( (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookup \
579	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithIrq \
580	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
581	: (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlb \
582	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq \
583	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
584	: (a_enmLabel) == kIemNativeLabelType_NonZeroRetOrPassUp \
585	? RT_BIT_32(IEMNATIVE_CALL_RET_GREG) \| RT_BIT_32(X86_GREG_xCX) /* <-- the difference */ \
586	: 0)
587	# else
588	# define IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmLabel) \
589	( (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookup \
590	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithIrq \
591	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
592	: (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlb \
593	\|\| (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq \
594	? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
595	: (a_enmLabel) == kIemNativeLabelType_NonZeroRetOrPassUp \
596	? RT_BIT_32(IEMNATIVE_CALL_RET_GREG) \| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
597	: 0)
598	#endif
599
600
601	/** Native code generator label definition. */
602	typedef struct IEMNATIVELABEL
603	{
604	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
605	* the epilog. */
606	uint32_t off;
607	/** The type of label (IEMNATIVELABELTYPE). */
608	uint16_t enmType;
609	/** Additional label data, type specific. */
610	uint16_t uData;
611	} IEMNATIVELABEL;
612	/** Pointer to a label. */
613	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
614
615
616
617	/** Native code generator fixup types. */
618	typedef enum
619	{
620	kIemNativeFixupType_Invalid = 0,
621	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
622	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
623	kIemNativeFixupType_Rel32,
624	#elif defined(RT_ARCH_ARM64)
625	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
626	kIemNativeFixupType_RelImm26At0,
627	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
628	kIemNativeFixupType_RelImm19At5,
629	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
630	kIemNativeFixupType_RelImm14At5,
631	#endif
632	kIemNativeFixupType_End
633	} IEMNATIVEFIXUPTYPE;
634
635	/** Native code generator fixup. */
636	typedef struct IEMNATIVEFIXUP
637	{
638	/** Code offset of the fixup location. */
639	uint32_t off;
640	/** The IEMNATIVELABEL this is a fixup for. */
641	uint16_t idxLabel;
642	/** The fixup type (IEMNATIVEFIXUPTYPE). */
643	uint8_t enmType;
644	/** Addend or other data. */
645	int8_t offAddend;
646	} IEMNATIVEFIXUP;
647	/** Pointer to a native code generator fixup. */
648	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
649
650
651
652	/** Native code generator fixup to per chunk TB tail code. */
653	typedef struct IEMNATIVEEXITFIXUP
654	{
655	/** Code offset of the fixup location. */
656	uint32_t off;
657	/** The exit reason. */
658	IEMNATIVELABELTYPE enmExitReason;
659	} IEMNATIVEEXITFIXUP;
660	/** Pointer to a native code generator TB exit fixup. */
661	typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
662
663	/**
664	* Per executable memory chunk context with addresses for common code.
665	*/
666	typedef struct IEMNATIVEPERCHUNKCTX
667	{
668	/** Pointers to the exit labels */
669	PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
670	} IEMNATIVEPERCHUNKCTX;
671	/** Pointer to per-chunk recompiler context. */
672	typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
673	/** Pointer to const per-chunk recompiler context. */
674	typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
675
676
677
678	/**
679	* One bit of the state.
680	*
681	* Each register state takes up two bits. We keep the two bits in two separate
682	* 64-bit words to simplify applying them to the guest shadow register mask in
683	* the register allocator.
684	*/
685	typedef union IEMLIVENESSBIT
686	{
687	uint64_t bm64;
688	RT_GCC_EXTENSION struct
689	{ /* bit no */
690	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
691	uint64_t fCr0 : 1; /*< 0x10 / 16: /
692	uint64_t fCr4 : 1; /*< 0x11 / 17: /
693	uint64_t fFcw : 1; /*< 0x12 / 18: /
694	uint64_t fFsw : 1; /*< 0x13 / 19: /
695	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
696	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
697	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
698	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
699	uint64_t fXcr0 : 1; /*< 0x2c / 44: /
700	uint64_t fMxCsr : 1; /*< 0x2d / 45: /
701	uint64_t fEflOther : 1; /*< 0x2e / 46: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
702	uint64_t fEflCf : 1; /*< 0x2f / 47: Carry flag (X86_EFL_CF / 0). /
703	uint64_t fEflPf : 1; /*< 0x30 / 48: Parity flag (X86_EFL_PF / 2). /
704	uint64_t fEflAf : 1; /*< 0x31 / 59: Auxilary carry flag (X86_EFL_AF / 4). /
705	uint64_t fEflZf : 1; /*< 0x32 / 50: Zero flag (X86_EFL_ZF / 6). /
706	uint64_t fEflSf : 1; /*< 0x33 / 51: Signed flag (X86_EFL_SF / 7). /
707	uint64_t fEflOf : 1; /*< 0x34 / 52: Overflow flag (X86_EFL_OF / 12). /
708	uint64_t fUnusedPc : 1; /*< 0x35 / 53: (PC in ) /
709	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
710	};
711	} IEMLIVENESSBIT;
712	AssertCompileSize(IEMLIVENESSBIT, 8);
713
714	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
715	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
716	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
717	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
718	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
719	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
720	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
721	#define IEMLIVENESSBIT_IDX_EFL_COUNT 7
722
723
724	/**
725	* A liveness state entry.
726	*
727	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
728	* Once we add a SSE register shadowing, we'll add another 64-bit element for
729	* that.
730	*/
731	typedef union IEMLIVENESSENTRY
732	{
733	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
734	uint64_t bm64[16 / 8];
735	uint16_t bm32[16 / 4];
736	uint16_t bm16[16 / 2];
737	uint8_t bm8[ 16 / 1];
738	IEMLIVENESSBIT aBits[2];
739	#else
740	uint64_t bm64[32 / 8];
741	uint16_t bm32[32 / 4];
742	uint16_t bm16[32 / 2];
743	uint8_t bm8[ 32 / 1];
744	IEMLIVENESSBIT aBits[4];
745	#endif
746	RT_GCC_EXTENSION struct
747	{
748	/** Bit \#0 of the register states. */
749	IEMLIVENESSBIT Bit0;
750	/** Bit \#1 of the register states. */
751	IEMLIVENESSBIT Bit1;
752	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
753	/** Bit \#2 of the register states. */
754	IEMLIVENESSBIT Bit2;
755	/** Bit \#3 of the register states. */
756	IEMLIVENESSBIT Bit3;
757	#endif
758	};
759	} IEMLIVENESSENTRY;
760	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
761	AssertCompileSize(IEMLIVENESSENTRY, 16);
762	#else
763	AssertCompileSize(IEMLIVENESSENTRY, 32);
764	#endif
765	/** Pointer to a liveness state entry. */
766	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
767	/** Pointer to a const liveness state entry. */
768	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
769
770	/** @name 64-bit value masks for IEMLIVENESSENTRY.
771	* @{ / / 0xzzzzyyyyxxxxwwww */
772	#define IEMLIVENESSBIT_MASK UINT64_C(0x001fffffffffffff)
773
774	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
775	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
776	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
777
778	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
779	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
780	#endif
781
782	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x001fc00000000000)
783	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x001f800000000000)
784
785	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
786	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
787	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
788	#endif
789	/** @} */
790
791
792	/** @name The liveness state for a register.
793	*
794	* The state values have been picked to with state accumulation in mind (what
795	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
796	* performance critical work done with the values.
797	*
798	* This is a compressed state that only requires 2 bits per register.
799	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
800	* 1. the incoming state from the following call,
801	* 2. the outgoing state for this call,
802	* 3. mask of the entries set in the 2nd.
803	*
804	* The mask entry (3rd one above) will be used both when updating the outgoing
805	* state and when merging in incoming state for registers not touched by the
806	* current call.
807	*
808	*
809	* Extended Layout:
810	*
811	* The extended layout variation differs from the above as it records the
812	* different register accesses as individual bits, and it is currently used for
813	* the delayed EFLAGS calculation experiments. The latter means that
814	* calls/tb-exits and potential calls/exceptions/tb-exits are recorded
815	* separately so the latter can be checked for in combination with clobbering.
816	*
817	* @{ */
818	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
819	/** The register will be clobbered and the current value thrown away.
820	*
821	* When this is applied to the state (2) we'll simply be AND'ing it with the
822	* (old) mask (3) and adding the register to the mask. This way we'll
823	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
824	* IEMLIVENESS_STATE_INPUT states. */
825	# define IEMLIVENESS_STATE_CLOBBERED 0
826	/** The register is unused in the remainder of the TB.
827	*
828	* This is an initial state and can not be set by any of the
829	* iemNativeLivenessFunc_xxxx callbacks. */
830	# define IEMLIVENESS_STATE_UNUSED 1
831	/** The register value is required in a potential call or exception.
832	*
833	* This means that the register value must be calculated and is best written to
834	* the state, but that any shadowing registers can be flushed thereafter as it's
835	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
836	*
837	* It is typically applied across the board, but we preserve incoming
838	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
839	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
840	* 1. r0 = old & ~mask;
841	* 2. r0 = t1 & (t1 >> 1);
842	* 3. state \|= r0 \| 0b10;
843	* 4. mask = ~0;
844	*/
845	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
846	/** The register value is used as input.
847	*
848	* This means that the register value must be calculated and it is best to keep
849	* it in a register. It does not need to be writtent out as such. This is the
850	* highest priority state.
851	*
852	* Whether the call modifies the register or not isn't relevant to earlier
853	* calls, so that's not recorded.
854	*
855	* When applying this state we just or in the value in the outgoing state and
856	* mask. */
857	# define IEMLIVENESS_STATE_INPUT 3
858	/** Mask of the state bits. */
859	# define IEMLIVENESS_STATE_MASK 3
860	/** The number of bits per state. */
861	# define IEMLIVENESS_STATE_BIT_COUNT 2
862
863	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state.
864	* @note only used in assertions. */
865	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
866	/** Check if we're expecting read accesses to a register with the given (previous) liveness state.
867	* @note only used in assertions. */
868	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
869	/** Check if a register clobbering is expected given the (previous) liveness state.
870	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
871	* include INPUT if the register is used in more than one place.
872	* @note only used in assertions. */
873	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
874
875	/** Check if all status flags are going to be clobbered and doesn't need
876	* calculating in the current step.
877	* @param a_pCurEntry The current liveness entry.
878	* @note Used by actual code. */
879	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
880	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
881
882	/***
883	* Construct a mask of what will be clobbered and never used.
884	*
885	* This is mainly used with IEMLIVENESSBIT_STATUS_EFL_MASK to avoid
886	* unnecessary EFLAGS calculations.
887	*
888	* @param a_pCurEntry The current liveness entry.
889	* @note Used by actual code.
890	*/
891	# define IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(a_pCurEntry) \
892	( ~((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_MASK )
893
894	/** Construct a mask of the guest registers in the UNUSED and XCPT_OR_CALL
895	* states, as these are no longer needed.
896	* @param a_pCurEntry The current liveness entry.
897	* @note Used by actual code. */
898	AssertCompile(IEMLIVENESS_STATE_UNUSED == 1 && IEMLIVENESS_STATE_XCPT_OR_CALL == 2);
899	# define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
900	( (a_pCurEntry)->Bit0.bm64 ^ (a_pCurEntry)->Bit1.bm64 )
901
902
903	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
904	/** The register is not used any more. */
905	# define IEMLIVENESS_STATE_UNUSED 0
906	/** Flag: The register is required in a potential call or/and exception. */
907	# define IEMLIVENESS_STATE_POTENTIAL_CALL 1
908	# define IEMLIVENESS_BIT_POTENTIAL_CALL 0
909	/** Flag: The register is read. */
910	# define IEMLIVENESS_STATE_READ 2
911	# define IEMLIVENESS_BIT_READ 1
912	/** Flag: The register is written. */
913	# define IEMLIVENESS_STATE_WRITE 4
914	# define IEMLIVENESS_BIT_WRITE 2
915	/** Flag: Unconditional call. */
916	# define IEMLIVENESS_STATE_CALL 8
917	# define IEMLIVENESS_BIT_CALL 3
918
919	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
920	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
921	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
922	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
923
924	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
925	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
926	&& !( ( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
927	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
928	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) \
929	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
930
931	/** Construct a mask of the registers not in the read or write state.
932	* @note We could skips writes, if they aren't from us, as this is just a hack
933	* to prevent trashing registers that have just been written or will be
934	* written when we retire the current instruction.
935	* @param a_pCurEntry The current liveness entry.
936	* @note Used by actual code. */
937	# define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
938	( ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
939	& ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
940	& IEMLIVENESSBIT_MASK )
941
942	/***
943	* Construct a mask of what will be clobbered and never used.
944	*
945	* This is mainly used with IEMLIVENESSBIT_STATUS_EFL_MASK to avoid
946	* unnecessary EFLAGS calculations.
947	*
948	* @param a_pCurEntry The current liveness entry.
949	* @note Used by actual code.
950	*/
951	# define IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(a_pCurEntry) \
952	( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
953	& ~( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
954	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
955	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) )
956
957	/**
958	* Construct a mask of what (EFLAGS) which can be postponed.
959	*
960	* The postponement is for the avoiding EFLAGS status bits calculations in the
961	* primary code stream whenever possible, and instead only do these in the TLB
962	* load and TB exit code paths which shouldn't be traveled quite as often.
963	* A requirement, though, is that the status bits will be clobbered later in the
964	* TB.
965	*
966	* User need to apply IEMLIVENESSBIT_STATUS_EFL_MASK if appropriate/necessary.
967	*
968	* @param a_pCurEntry The current liveness entry.
969	* @note Used by actual code.
970	*/
971	# define IEMLIVENESS_STATE_GET_CAN_BE_POSTPONED_SET(a_pCurEntry) \
972	( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
973	& (a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
974	& ~( (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
975	\| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) )
976
977	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
978	/** @} */
979
980	/** @name Liveness helpers for builtin functions and similar.
981	*
982	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
983	* own set of manipulator macros for those.
984	*
985	* @{ */
986	/** Initializing the state as all unused. */
987	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
988	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
989	do { \
990	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
991	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
992	} while (0)
993	#else
994	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
995	do { \
996	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
997	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
998	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
999	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
1000	} while (0)
1001	#endif
1002
1003	/** Initializing the outgoing state with a potential xcpt or call state.
1004	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT.
1005	*
1006	* @note Must invoke IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL when done!
1007	*/
1008	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1009	# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1010	do { \
1011	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
1012	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
1013	} while (0)
1014	#else
1015	# define IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1016	do { \
1017	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
1018	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
1019	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
1020	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = 0; \
1021	} while (0)
1022	#endif
1023
1024	/** Completes IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL after applying any
1025	* other state modifications.
1026	*/
1027	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1028	# define IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) ((void)0)
1029	#else
1030	# define IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1031	do { \
1032	uint64_t const fInhMask = ~( (a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL].bm64 \
1033	\| (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE].bm64); \
1034	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 & fInhMask; \
1035	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64 & fInhMask; \
1036	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & fInhMask; \
1037	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 \|= (a_pIncoming)->aBits[IEMLIVENESS_BIT_CALL].bm64 & fInhMask; \
1038	} while (0)
1039	#endif
1040
1041	/** Initializing the outgoing state with an unconditional call state.
1042	* This should only really be used alone. */
1043	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1044	# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
1045	do { \
1046	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
1047	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
1048	} while (0)
1049	#else
1050	# define IEM_LIVENESS_RAW_INIT_WITH_CALL(a_pOutgoing, a_pIncoming) \
1051	do { \
1052	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
1053	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = 0; \
1054	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
1055	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
1056	RT_NOREF(a_pIncoming); \
1057	} while (0)
1058	#endif
1059
1060	#if 0 /* unused */
1061	/** Initializing the outgoing state with an unconditional call state as well as
1062	* an potential call/exception preceeding it.
1063	* This should only really be used alone. */
1064	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1065	# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1066	do { \
1067	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
1068	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
1069	} while (0)
1070	#else
1071	# define IEM_LIVENESS_RAW_INIT_WITH_CALL_AND_POTENTIAL_CALL(a_pOutgoing, a_pIncoming) \
1072	do { \
1073	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 = IEMLIVENESSBIT_MASK; \
1074	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_CALL ].bm64 = IEMLIVENESSBIT_MASK; \
1075	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
1076	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
1077	} while (0)
1078	#endif
1079	#endif
1080
1081	/** Adds a segment base register as input to the outgoing state. */
1082	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1083	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
1084	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
1085	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
1086	} while (0)
1087	#else
1088	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
1089	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
1090	} while (0)
1091	#endif
1092
1093	/** Adds a segment attribute register as input to the outgoing state. */
1094	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1095	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
1096	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
1097	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
1098	} while (0)
1099	#else
1100	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
1101	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
1102	} while (0)
1103	#endif
1104
1105	/** Adds a segment limit register as input to the outgoing state. */
1106	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1107	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
1108	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
1109	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
1110	} while (0)
1111	#else
1112	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
1113	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
1114	} while (0)
1115	#endif
1116
1117	/** Adds a segment limit register as input to the outgoing state. */
1118	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
1119	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
1120	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
1121	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
1122	} while (0)
1123	#else
1124	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
1125	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
1126	} while (0)
1127	#endif
1128	/** @} */
1129
1130	/** @def IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY
1131	* Debug assertion that the required flags are available and not incorrectly skipped.
1132	*/
1133	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
1134	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded) \
1135	AssertMsg(!((a_pReNative)->fSkippingEFlags & (a_fEflNeeded)), \
1136	("%#x & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, \
1137	a_fEflNeeded, (a_pReNative)->fSkippingEFlags & (a_fEflNeeded) ))
1138	#else
1139	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded) ((void)0)
1140	#endif
1141
1142	/** @def IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY
1143	* Debug assertion that the required flags are available and not incorrectly postponed.
1144	*/
1145	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1146	# define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) \
1147	AssertMsg(!((a_pReNative)->PostponedEfl.fEFlags & (a_fEflNeeded)), \
1148	("%#x & %#x -> %#x\n", (a_pReNative)->PostponedEfl.fEFlags, \
1149	a_fEflNeeded, (a_pReNative)->PostponedEfl.fEFlags & (a_fEflNeeded) ))
1150	#else
1151	# define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) ((void)0)
1152	#endif
1153
1154	/** @def IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING
1155	* Debug assertion that the required flags are available and not incorrectly
1156	* skipped or postponed.
1157	*/
1158	#if defined(IEMNATIVE_WITH_EFLAGS_SKIPPING) && defined(IEMNATIVE_WITH_EFLAGS_POSTPONING)
1159	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
1160	AssertMsg(!(((a_pReNative)->fSkippingEFlags \| (a_pReNative)->PostponedEfl.fEFlags) & (a_fEflNeeded)), \
1161	("(%#x \| %#x) & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, (a_pReNative)->PostponedEfl.fEFlags, \
1162	a_fEflNeeded, ((a_pReNative)->fSkippingEFlags \| (a_pReNative)->PostponedEfl.fEFlags) & (a_fEflNeeded) ))
1163	#elif defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)
1164	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
1165	IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded)
1166	#elif defined(IEMNATIVE_WITH_EFLAGS_POSTPONING) \
1167	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
1168	IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded)
1169	#else
1170	# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) ((void)0)
1171	#endif
1172
1173	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
1174	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
1175	* calculated and up to date. This is to double check that we haven't skipped
1176	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
1177	* @note has to be placed in
1178	*/
1179	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1180	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { \
1181	(a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); \
1182	} while (0)
1183	#else
1184	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
1185	#endif
1186
1187
1188	/** @def IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS
1189	* Number of extra instructions to allocate for each TB exit to account for
1190	* postponed EFLAGS calculations.
1191	*/
1192	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1193	# ifdef RT_ARCH_AMD64
1194	# ifdef VBOX_STRICT
1195	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 64
1196	# else
1197	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 32
1198	# endif
1199	# elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
1200	# ifdef VBOX_STRICT
1201	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 48
1202	# else
1203	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 32
1204	# endif
1205	# else
1206	# error "port me"
1207	# endif
1208	#else
1209	# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS 0
1210	#endif
1211
1212	/** @def IEMNATIVE_CLEAR_POSTPONED_EFLAGS
1213	* Helper macro function for calling iemNativeClearPostponedEFlags() when
1214	* IEMNATIVE_WITH_EFLAGS_POSTPONING is enabled.
1215	*/
1216	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1217	# define IEMNATIVE_CLEAR_POSTPONED_EFLAGS(a_pReNative, a_fEflClobbered) iemNativeClearPostponedEFlags<a_fEflClobbered>(a_pReNative)
1218	#else
1219	# define IEMNATIVE_CLEAR_POSTPONED_EFLAGS(a_pReNative, a_fEflClobbered) ((void)0)
1220	#endif
1221
1222	/** @def IEMNATIVE_HAS_POSTPONED_EFLAGS_CALCS
1223	* Macro for testing whether there are currently any postponed EFLAGS calcs w/o
1224	* needing to \#ifdef the check.
1225	*/
1226	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1227	# define IEMNATIVE_HAS_POSTPONED_EFLAGS_CALCS(a_pReNative) ((a_pReNative)->PostponedEfl.fEFlags != 0)
1228	#else
1229	# define IEMNATIVE_HAS_POSTPONED_EFLAGS_CALCS(a_pReNative) false
1230	#endif
1231
1232
1233	/**
1234	* Translation block debug info entry type.
1235	*/
1236	typedef enum IEMTBDBGENTRYTYPE
1237	{
1238	kIemTbDbgEntryType_Invalid = 0,
1239	/** The entry is for marking a native code position.
1240	* Entries following this all apply to this position. */
1241	kIemTbDbgEntryType_NativeOffset,
1242	/** The entry is for a new guest instruction. */
1243	kIemTbDbgEntryType_GuestInstruction,
1244	/** Marks the start of a threaded call. */
1245	kIemTbDbgEntryType_ThreadedCall,
1246	/** Marks the location of a label. */
1247	kIemTbDbgEntryType_Label,
1248	/** Info about a host register shadowing a guest register. */
1249	kIemTbDbgEntryType_GuestRegShadowing,
1250	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1251	/** Info about a host SIMD register shadowing a guest SIMD register. */
1252	kIemTbDbgEntryType_GuestSimdRegShadowing,
1253	#endif
1254	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1255	/** Info about a delayed RIP update. */
1256	kIemTbDbgEntryType_DelayedPcUpdate,
1257	#endif
1258	#if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1259	/** Info about a shadowed guest register becoming dirty. */
1260	kIemTbDbgEntryType_GuestRegDirty,
1261	/** Info about register writeback/flush oepration. */
1262	kIemTbDbgEntryType_GuestRegWriteback,
1263	#endif
1264	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1265	/** Info about a delayed EFLAGS calculation. */
1266	kIemTbDbgEntryType_PostponedEFlagsCalc,
1267	#endif
1268	kIemTbDbgEntryType_End
1269	} IEMTBDBGENTRYTYPE;
1270
1271	/**
1272	* Translation block debug info entry.
1273	*/
1274	typedef union IEMTBDBGENTRY
1275	{
1276	/** Plain 32-bit view. */
1277	uint32_t u;
1278
1279	/** Generic view for getting at the type field. */
1280	struct
1281	{
1282	/** IEMTBDBGENTRYTYPE */
1283	uint32_t uType : 4;
1284	uint32_t uTypeSpecific : 28;
1285	} Gen;
1286
1287	struct
1288	{
1289	/** kIemTbDbgEntryType_ThreadedCall1. */
1290	uint32_t uType : 4;
1291	/** Native code offset. */
1292	uint32_t offNative : 28;
1293	} NativeOffset;
1294
1295	struct
1296	{
1297	/** kIemTbDbgEntryType_GuestInstruction. */
1298	uint32_t uType : 4;
1299	uint32_t uUnused : 4;
1300	/** The IEM_F_XXX flags. */
1301	uint32_t fExec : 24;
1302	} GuestInstruction;
1303
1304	struct
1305	{
1306	/* kIemTbDbgEntryType_ThreadedCall. */
1307	uint32_t uType : 4;
1308	/** Set if the call was recompiled to native code, clear if just calling
1309	* threaded function. */
1310	uint32_t fRecompiled : 1;
1311	uint32_t uUnused : 11;
1312	/** The threaded call number (IEMTHREADEDFUNCS). */
1313	uint32_t enmCall : 16;
1314	} ThreadedCall;
1315
1316	struct
1317	{
1318	/* kIemTbDbgEntryType_Label. */
1319	uint32_t uType : 4;
1320	uint32_t uUnused : 4;
1321	/** The label type (IEMNATIVELABELTYPE). */
1322	uint32_t enmLabel : 8;
1323	/** The label data. */
1324	uint32_t uData : 16;
1325	} Label;
1326
1327	struct
1328	{
1329	/* kIemTbDbgEntryType_GuestRegShadowing. */
1330	uint32_t uType : 4;
1331	uint32_t uUnused : 4;
1332	/** The guest register being shadowed (IEMNATIVEGSTREG). */
1333	uint32_t idxGstReg : 8;
1334	/** The host new register number, UINT8_MAX if dropped. */
1335	uint32_t idxHstReg : 8;
1336	/** The previous host register number, UINT8_MAX if new. */
1337	uint32_t idxHstRegPrev : 8;
1338	} GuestRegShadowing;
1339
1340	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1341	struct
1342	{
1343	/* kIemTbDbgEntryType_GuestSimdRegShadowing. */
1344	uint32_t uType : 4;
1345	uint32_t uUnused : 4;
1346	/** The guest register being shadowed (IEMNATIVEGSTSIMDREG). */
1347	uint32_t idxGstSimdReg : 8;
1348	/** The host new register number, UINT8_MAX if dropped. */
1349	uint32_t idxHstSimdReg : 8;
1350	/** The previous host register number, UINT8_MAX if new. */
1351	uint32_t idxHstSimdRegPrev : 8;
1352	} GuestSimdRegShadowing;
1353	#endif
1354
1355	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1356	struct
1357	{
1358	/* kIemTbDbgEntryType_DelayedPcUpdate. */
1359	uint32_t uType : 4;
1360	/** Number of instructions skipped. */
1361	uint32_t cInstrSkipped : 8;
1362	/* The instruction offset added to the program counter. */
1363	int32_t offPc : 20;
1364	} DelayedPcUpdate;
1365	#endif
1366
1367	#if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1368	struct
1369	{
1370	/* kIemTbDbgEntryType_GuestRegDirty. */
1371	uint32_t uType : 4;
1372	uint32_t uUnused : 11;
1373	/** Flag whether this is about a SIMD (true) or general (false) register. */
1374	uint32_t fSimdReg : 1;
1375	/** The guest register index being marked as dirty. */
1376	uint32_t idxGstReg : 8;
1377	/** The host register number this register is shadowed in .*/
1378	uint32_t idxHstReg : 8;
1379	} GuestRegDirty;
1380
1381	struct
1382	{
1383	/* kIemTbDbgEntryType_GuestRegWriteback. */
1384	uint32_t uType : 4;
1385	/** Flag whether this is about a SIMD (true) or general (false) register flush. */
1386	uint32_t fSimdReg : 1;
1387	/** The mask shift. */
1388	uint32_t cShift : 2;
1389	/** The guest register mask being written back. */
1390	uint32_t fGstReg : 25;
1391	} GuestRegWriteback;
1392	#endif
1393
1394	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1395	struct
1396	{
1397	/* kIemTbDbgEntryType_PostponedEFlagsCalc. */
1398	uint32_t uType : 4;
1399	/** The EFLAGS operation (IEMNATIVE_POSTPONED_EFL_OP_T). */
1400	uint32_t enmOp : 4;
1401	/** The mask shift. */
1402	uint32_t cOpBits : 8;
1403	/** The emit instance number (0-based). */
1404	uint32_t idxEmit : 8;
1405	/** Unused. */
1406	uint32_t uUnused : 8;
1407	} PostponedEflCalc;
1408	#endif
1409	} IEMTBDBGENTRY;
1410	AssertCompileSize(IEMTBDBGENTRY, sizeof(uint32_t));
1411	/** Pointer to a debug info entry. */
1412	typedef IEMTBDBGENTRY *PIEMTBDBGENTRY;
1413	/** Pointer to a const debug info entry. */
1414	typedef IEMTBDBGENTRY const *PCIEMTBDBGENTRY;
1415
1416	/**
1417	* Translation block debug info.
1418	*/
1419	typedef struct IEMTBDBG
1420	{
1421	/** This is the flat PC corresponding to IEMTB::GCPhysPc. */
1422	RTGCPTR FlatPc;
1423	/** Number of entries in aEntries. */
1424	uint32_t cEntries;
1425	/** The offset of the last kIemTbDbgEntryType_NativeOffset record. */
1426	uint32_t offNativeLast;
1427	/** Debug info entries. */
1428	RT_FLEXIBLE_ARRAY_EXTENSION
1429	IEMTBDBGENTRY aEntries[RT_FLEXIBLE_ARRAY];
1430	} IEMTBDBG;
1431	/** Pointer to TB debug info. */
1432	typedef IEMTBDBG *PIEMTBDBG;
1433	/** Pointer to const TB debug info. */
1434	typedef IEMTBDBG const *PCIEMTBDBG;
1435
1436	/**
1437	* Guest registers that can be shadowed in GPRs.
1438	*
1439	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
1440	* must be placed last, as the liveness state tracks it as 7 subcomponents and
1441	* we don't want to waste space here.
1442	*
1443	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
1444	* friends as well as IEMAllN8veLiveness.cpp.
1445	*/
1446	typedef enum IEMNATIVEGSTREG : uint8_t
1447	{
1448	kIemNativeGstReg_GprFirst = 0,
1449	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
1450	kIemNativeGstReg_Cr0,
1451	kIemNativeGstReg_Cr4,
1452	kIemNativeGstReg_FpuFcw,
1453	kIemNativeGstReg_FpuFsw,
1454	kIemNativeGstReg_SegBaseFirst,
1455	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
1456	kIemNativeGstReg_SegAttribFirst,
1457	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
1458	kIemNativeGstReg_SegLimitFirst,
1459	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
1460	kIemNativeGstReg_SegSelFirst,
1461	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
1462	kIemNativeGstReg_Xcr0,
1463	kIemNativeGstReg_MxCsr,
1464	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags. /
1465	/* 6 entry gap for liveness EFlags subdivisions. */
1466	kIemNativeGstReg_Pc = kIemNativeGstReg_EFlags + 7,
1467	kIemNativeGstReg_End
1468	} IEMNATIVEGSTREG;
1469	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
1470	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
1471	AssertCompile(RT_BIT_64(kIemNativeGstReg_Pc) - UINT64_C(1) == IEMLIVENESSBIT_MASK);
1472
1473	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
1474	* @{ */
1475	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
1476	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
1477	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
1478	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
1479	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
1480	/** @} */
1481
1482	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1483
1484	/**
1485	* Guest registers that can be shadowed in host SIMD registers.
1486	*
1487	* @todo r=aeichner Liveness tracking
1488	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
1489	*/
1490	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
1491	{
1492	kIemNativeGstSimdReg_SimdRegFirst = 0,
1493	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
1494	kIemNativeGstSimdReg_End
1495	} IEMNATIVEGSTSIMDREG;
1496
1497	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
1498	* @{ */
1499	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
1500	/** @} */
1501
1502	/**
1503	* The Load/store size for a SIMD guest register.
1504	*/
1505	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
1506	{
1507	/** Invalid size. */
1508	kIemNativeGstSimdRegLdStSz_Invalid = 0,
1509	/** Loads the low 128-bit of a guest SIMD register. */
1510	kIemNativeGstSimdRegLdStSz_Low128,
1511	/** Loads the high 128-bit of a guest SIMD register. */
1512	kIemNativeGstSimdRegLdStSz_High128,
1513	/** Loads the whole 256-bits of a guest SIMD register. */
1514	kIemNativeGstSimdRegLdStSz_256,
1515	/** End value. */
1516	kIemNativeGstSimdRegLdStSz_End
1517	} IEMNATIVEGSTSIMDREGLDSTSZ;
1518
1519	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
1520
1521	/**
1522	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
1523	*/
1524	typedef enum IEMNATIVEGSTREGUSE
1525	{
1526	/** The usage is read-only, the register holding the guest register
1527	* shadow copy will not be modified by the caller. */
1528	kIemNativeGstRegUse_ReadOnly = 0,
1529	/** The caller will update the guest register (think: PC += cbInstr).
1530	* The guest shadow copy will follow the returned register. */
1531	kIemNativeGstRegUse_ForUpdate,
1532	/** The call will put an entirely new value in the guest register, so
1533	* if new register is allocate it will be returned uninitialized. */
1534	kIemNativeGstRegUse_ForFullWrite,
1535	/** The caller will use the guest register value as input in a calculation
1536	* and the host register will be modified.
1537	* This means that the returned host register will not be marked as a shadow
1538	* copy of the guest register. */
1539	kIemNativeGstRegUse_Calculation
1540	} IEMNATIVEGSTREGUSE;
1541
1542	/**
1543	* Guest registers (classes) that can be referenced.
1544	*/
1545	typedef enum IEMNATIVEGSTREGREF : uint8_t
1546	{
1547	kIemNativeGstRegRef_Invalid = 0,
1548	kIemNativeGstRegRef_Gpr,
1549	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
1550	kIemNativeGstRegRef_EFlags,
1551	kIemNativeGstRegRef_MxCsr,
1552	kIemNativeGstRegRef_FpuReg,
1553	kIemNativeGstRegRef_MReg,
1554	kIemNativeGstRegRef_XReg,
1555	kIemNativeGstRegRef_X87,
1556	kIemNativeGstRegRef_XState,
1557	//kIemNativeGstRegRef_YReg, - doesn't work.
1558	kIemNativeGstRegRef_End
1559	} IEMNATIVEGSTREGREF;
1560
1561
1562	/** Variable kinds. */
1563	typedef enum IEMNATIVEVARKIND : uint8_t
1564	{
1565	/** Customary invalid zero value. */
1566	kIemNativeVarKind_Invalid = 0,
1567	/** This is either in a register or on the stack. */
1568	kIemNativeVarKind_Stack,
1569	/** Immediate value - loaded into register when needed, or can live on the
1570	* stack if referenced (in theory). */
1571	kIemNativeVarKind_Immediate,
1572	/** Variable reference - loaded into register when needed, never stack. */
1573	kIemNativeVarKind_VarRef,
1574	/** Guest register reference - loaded into register when needed, never stack. */
1575	kIemNativeVarKind_GstRegRef,
1576	/** End of valid values. */
1577	kIemNativeVarKind_End
1578	} IEMNATIVEVARKIND;
1579
1580
1581	/** Variable or argument. */
1582	typedef struct IEMNATIVEVAR
1583	{
1584	/** The kind of variable. */
1585	IEMNATIVEVARKIND enmKind;
1586	/** The variable size in bytes. */
1587	uint8_t cbVar;
1588	/** The first stack slot (uint64_t), except for immediate and references
1589	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1590	* has a stack slot it has been initialized and has a value. Unused variables
1591	* has neither a stack slot nor a host register assignment. */
1592	uint8_t idxStackSlot;
1593	/** The host register allocated for the variable, UINT8_MAX if not. */
1594	uint8_t idxReg;
1595	/** The argument number if argument, UINT8_MAX if regular variable. */
1596	uint8_t uArgNo;
1597	/** If referenced, the index (unpacked) of the variable referencing this one,
1598	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1599	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1600	uint8_t idxReferrerVar;
1601	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1602	* @todo not sure what this really is for... */
1603	IEMNATIVEGSTREG enmGstReg;
1604	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1605	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1606	* only valid when idxReg is not UINT8_MAX. */
1607	bool fSimdReg : 1;
1608	/** Set if the registered is currently used exclusively, false if the
1609	* variable is idle and the register can be grabbed. */
1610	bool fRegAcquired : 1;
1611	#else
1612	/** Set if the registered is currently used exclusively, false if the
1613	* variable is idle and the register can be grabbed. */
1614	bool fRegAcquired;
1615	#endif
1616
1617	union
1618	{
1619	/** kIemNativeVarKind_Immediate: The immediate value. */
1620	uint64_t uValue;
1621	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1622	uint8_t idxRefVar;
1623	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1624	struct
1625	{
1626	/** The class of register. */
1627	IEMNATIVEGSTREGREF enmClass;
1628	/** Index within the class. */
1629	uint8_t idx;
1630	} GstRegRef;
1631	} u;
1632	} IEMNATIVEVAR;
1633	/** Pointer to a variable or argument. */
1634	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1635	/** Pointer to a const variable or argument. */
1636	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1637
1638	/** What is being kept in a host register. */
1639	typedef enum IEMNATIVEWHAT : uint8_t
1640	{
1641	/** The traditional invalid zero value. */
1642	kIemNativeWhat_Invalid = 0,
1643	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1644	kIemNativeWhat_Var,
1645	/** Temporary register, this is typically freed when a MC completes. */
1646	kIemNativeWhat_Tmp,
1647	/** Call argument w/o a variable mapping. This is free (via
1648	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1649	kIemNativeWhat_Arg,
1650	/** Return status code.
1651	* @todo not sure if we need this... */
1652	kIemNativeWhat_rc,
1653	/** The fixed pVCpu (PVMCPUCC) register.
1654	* @todo consider offsetting this on amd64 to use negative offsets to access
1655	* more members using 8-byte disp. */
1656	kIemNativeWhat_pVCpuFixed,
1657	/** The fixed pCtx (PCPUMCTX) register.
1658	* @todo consider offsetting this on amd64 to use negative offsets to access
1659	* more members using 8-byte disp. */
1660	kIemNativeWhat_pCtxFixed,
1661	/** Fixed temporary register. */
1662	kIemNativeWhat_FixedTmp,
1663	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1664	/** Shadow RIP for the delayed RIP updating debugging. */
1665	kIemNativeWhat_PcShadow,
1666	#endif
1667	/** Register reserved by the CPU or OS architecture. */
1668	kIemNativeWhat_FixedReserved,
1669	/** End of valid values. */
1670	kIemNativeWhat_End
1671	} IEMNATIVEWHAT;
1672
1673	/**
1674	* Host general register entry.
1675	*
1676	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1677	*
1678	* @todo Track immediate values in host registers similarlly to how we track the
1679	* guest register shadow copies. For it to be real helpful, though,
1680	* we probably need to know which will be reused and put them into
1681	* non-volatile registers, otherwise it's going to be more or less
1682	* restricted to an instruction or two.
1683	*/
1684	typedef struct IEMNATIVEHSTREG
1685	{
1686	/** Set of guest registers this one shadows.
1687	*
1688	* Using a bitmap here so we can designate the same host register as a copy
1689	* for more than one guest register. This is expected to be useful in
1690	* situations where one value is copied to several registers in a sequence.
1691	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1692	* sequence we'd want to let this register follow to be a copy of and there
1693	* will always be places where we'd be picking the wrong one.
1694	*/
1695	uint64_t fGstRegShadows;
1696	/** What is being kept in this register. */
1697	IEMNATIVEWHAT enmWhat;
1698	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1699	uint8_t idxVar;
1700	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1701	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1702	* that scope. */
1703	uint8_t idxStackSlot;
1704	/** Alignment padding. */
1705	uint8_t abAlign[5];
1706	} IEMNATIVEHSTREG;
1707
1708
1709	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1710	/**
1711	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1712	* halves, on architectures where there is no 256-bit register available this entry will track
1713	* two adjacent 128-bit host registers.
1714	*
1715	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1716	*/
1717	typedef struct IEMNATIVEHSTSIMDREG
1718	{
1719	/** Set of guest registers this one shadows.
1720	*
1721	* Using a bitmap here so we can designate the same host register as a copy
1722	* for more than one guest register. This is expected to be useful in
1723	* situations where one value is copied to several registers in a sequence.
1724	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1725	* sequence we'd want to let this register follow to be a copy of and there
1726	* will always be places where we'd be picking the wrong one.
1727	*/
1728	uint64_t fGstRegShadows;
1729	/** What is being kept in this register. */
1730	IEMNATIVEWHAT enmWhat;
1731	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1732	uint8_t idxVar;
1733	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1734	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1735	/** Alignment padding. */
1736	uint8_t abAlign[5];
1737	} IEMNATIVEHSTSIMDREG;
1738	#endif
1739
1740
1741	/**
1742	* Core state for the native recompiler, that is, things that needs careful
1743	* handling when dealing with branches.
1744	*/
1745	typedef struct IEMNATIVECORESTATE
1746	{
1747	/** Allocation bitmap for aHstRegs. */
1748	uint32_t bmHstRegs;
1749
1750	/** Bitmap marking which host register contains guest register shadow copies.
1751	* This is used during register allocation to try preserve copies. */
1752	uint32_t bmHstRegsWithGstShadow;
1753	/** Bitmap marking valid entries in aidxGstRegShadows. */
1754	uint64_t bmGstRegShadows;
1755	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1756	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1757	uint64_t bmGstRegShadowDirty;
1758	#endif
1759
1760	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1761	/** The current instruction offset in bytes from when the guest program counter
1762	* was updated last. Used for delaying the write to the guest context program counter
1763	* as long as possible. */
1764	int64_t offPc;
1765	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
1766	/** Set after we've loaded PC into uPcUpdatingDebug at the first update. */
1767	bool fDebugPcInitialized;
1768	# endif
1769	#endif
1770
1771	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1772	/** Allocation bitmap for aHstSimdRegs. */
1773	uint32_t bmHstSimdRegs;
1774
1775	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1776	* This is used during register allocation to try preserve copies. */
1777	uint32_t bmHstSimdRegsWithGstShadow;
1778	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1779	uint64_t bmGstSimdRegShadows;
1780	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1781	uint64_t bmGstSimdRegShadowDirtyLo128;
1782	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1783	uint64_t bmGstSimdRegShadowDirtyHi128;
1784	#endif
1785
1786	union
1787	{
1788	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1789	uint8_t aidxArgVars[8];
1790	/** For more efficient resetting. */
1791	uint64_t u64ArgVars;
1792	};
1793
1794	/** Allocation bitmap for the stack. */
1795	uint32_t bmStack;
1796	/** Allocation bitmap for aVars. */
1797	uint32_t bmVars;
1798
1799	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1800	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1801	* (A shadow copy of a guest register can only be held in a one host register,
1802	* there are no duplicate copies or ambiguities like that). */
1803	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1804	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1805	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1806	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1807	* (A shadow copy of a guest register can only be held in a one host register,
1808	* there are no duplicate copies or ambiguities like that). */
1809	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1810	#endif
1811
1812	/** Host register allocation tracking. */
1813	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1814	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1815	/** Host SIMD register allocation tracking. */
1816	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1817	#endif
1818
1819	/** Variables and arguments. */
1820	IEMNATIVEVAR aVars[9];
1821	} IEMNATIVECORESTATE;
1822	/** Pointer to core state. */
1823	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1824	/** Pointer to const core state. */
1825	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1826
1827	/** @def IEMNATIVE_VAR_IDX_UNPACK
1828	* @returns Index into IEMNATIVECORESTATE::aVars.
1829	* @param a_idxVar Variable index w/ magic (in strict builds).
1830	*/
1831	/** @def IEMNATIVE_VAR_IDX_PACK
1832	* @returns Variable index w/ magic (in strict builds).
1833	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1834	*/
1835	#ifdef VBOX_STRICT
1836	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1837	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1838	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1839	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1840	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1841	#else
1842	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1843	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1844	#endif
1845
1846
1847	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1848	/** Clear the dirty state of the given guest SIMD register. */
1849	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1850	do { \
1851	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1852	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1853	} while (0)
1854
1855	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1856	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1857	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1858	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1859	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1860	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1861	/** Returns whether the given guest SIMD register is dirty. */
1862	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1863	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1864
1865	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1866	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1867	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1868	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1869	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1870	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1871
1872	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1873	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1874	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1875	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1876	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1877	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1878	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1879	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1880	# ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1881	/** Flag indicating that the guest MXCSR was synced to the host floating point control register. */
1882	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED RT_BIT_32(4)
1883	/** Flag indicating whether the host floating point control register was saved before overwriting it. */
1884	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SAVED RT_BIT_32(5)
1885	# endif
1886	#endif
1887
1888
1889	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1890	typedef enum IEMNATIVE_POSTPONED_EFL_OP_T : uint8_t
1891	{
1892	kIemNativePostponedEflOp_Invalid = 0,
1893	/** Logical operation.
1894	* Operands: result register.
1895	* @note This clears OF, CF and (undefined) AF, thus no need for inputs. */
1896	kIemNativePostponedEflOp_Logical,
1897	kIemNativePostponedEflOp_End
1898	} IEMNATIVE_POSTPONED_EFL_OP_T;
1899	#endif /* IEMNATIVE_WITH_EFLAGS_POSTPONING */
1900
1901	/**
1902	* Conditional stack entry.
1903	*/
1904	typedef struct IEMNATIVECOND
1905	{
1906	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1907	bool fInElse;
1908	union
1909	{
1910	RT_GCC_EXTENSION struct
1911	{
1912	/** Set if the if-block unconditionally exited the TB. */
1913	bool fIfExitTb;
1914	/** Set if the else-block unconditionally exited the TB. */
1915	bool fElseExitTb;
1916	};
1917	/** Indexed by fInElse. */
1918	bool afExitTb[2];
1919	};
1920	bool afPadding[5];
1921	/** The label for the IEM_MC_ELSE. */
1922	uint32_t idxLabelElse;
1923	/** The label for the IEM_MC_ENDIF. */
1924	uint32_t idxLabelEndIf;
1925	/** The initial state snapshot as the if-block starts executing. */
1926	IEMNATIVECORESTATE InitialState;
1927	/** The state snapshot at the end of the if-block. */
1928	IEMNATIVECORESTATE IfFinalState;
1929	} IEMNATIVECOND;
1930	/** Pointer to a condition stack entry. */
1931	typedef IEMNATIVECOND *PIEMNATIVECOND;
1932
1933
1934	/**
1935	* Native recompiler state.
1936	*/
1937	typedef struct IEMRECOMPILERSTATE
1938	{
1939	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1940	* IEMNATIVEINSTR units. */
1941	uint32_t cInstrBufAlloc;
1942	#ifdef VBOX_STRICT
1943	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1944	uint32_t offInstrBufChecked;
1945	#else
1946	uint32_t uPadding1; /* We don't keep track of the size here... */
1947	#endif
1948	/** Fixed temporary code buffer for native recompilation. */
1949	PIEMNATIVEINSTR pInstrBuf;
1950
1951	/** Bitmaps with the label types used. */
1952	uint64_t bmLabelTypes;
1953	/** Actual number of labels in paLabels. */
1954	uint32_t cLabels;
1955	/** Max number of entries allowed in paLabels before reallocating it. */
1956	uint32_t cLabelsAlloc;
1957	/** Labels defined while recompiling (referenced by fixups). */
1958	PIEMNATIVELABEL paLabels;
1959	/** Array with indexes of unique labels (uData always 0). */
1960	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1961
1962	/** Actual number of fixups paFixups. */
1963	uint32_t cFixups;
1964	/** Max number of entries allowed in paFixups before reallocating it. */
1965	uint32_t cFixupsAlloc;
1966	/** Buffer used by the recompiler for recording fixups when generating code. */
1967	PIEMNATIVEFIXUP paFixups;
1968
1969	/** Actual number of fixups in paTbExitFixups. */
1970	uint32_t cTbExitFixups;
1971	/** Max number of entries allowed in paTbExitFixups before reallocating it. */
1972	uint32_t cTbExitFixupsAlloc;
1973	/** Buffer used by the recompiler for recording fixups when generating code. */
1974	PIEMNATIVEEXITFIXUP paTbExitFixups;
1975
1976	#if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) \|\| defined(VBOX_WITH_STATISTICS)
1977	/** Statistics: The idxInstr+1 value at the last PC update. */
1978	uint8_t idxInstrPlusOneOfLastPcUpdate;
1979	#endif
1980
1981	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1982	/** Number of debug info entries allocated for pDbgInfo. */
1983	uint32_t cDbgInfoAlloc;
1984	/** Debug info. */
1985	PIEMTBDBG pDbgInfo;
1986	#endif
1987
1988	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1989	/** The current call index (liveness array and threaded calls in TB). */
1990	uint32_t idxCurCall;
1991	/** Number of liveness entries allocated. */
1992	uint32_t cLivenessEntriesAlloc;
1993	/** Liveness entries for all the calls in the TB begin recompiled.
1994	* The entry for idxCurCall contains the info for what the next call will
1995	* require wrt registers. (Which means the last entry is the initial liveness
1996	* state.) */
1997	PIEMLIVENESSENTRY paLivenessEntries;
1998	#endif
1999
2000	/** The translation block being recompiled. */
2001	PCIEMTB pTbOrg;
2002	/** The VMCPU structure of the EMT. */
2003	PVMCPUCC pVCpu;
2004
2005	/** Condition sequence number (for generating unique labels). */
2006	uint16_t uCondSeqNo;
2007	/** Check IRQ sequence number (for generating unique labels). */
2008	uint16_t uCheckIrqSeqNo;
2009	/** TLB load sequence number (for generating unique labels). */
2010	uint16_t uTlbSeqNo;
2011	/** The current condition stack depth (aCondStack). */
2012	uint8_t cCondDepth;
2013
2014	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
2015	uint8_t cArgsX;
2016	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
2017	uint32_t fCImpl;
2018	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
2019	uint32_t fMc;
2020	/** The expected IEMCPU::fExec value for the current call/instruction. */
2021	uint32_t fExec;
2022	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2023	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
2024	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
2025	*
2026	* This is an optimization because these control registers can only be changed from
2027	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
2028	* consisting of multiple SIMD instructions.
2029	*/
2030	uint32_t fSimdRaiseXcptChecksEmitted;
2031	#endif
2032	/** The call number of the last CheckIrq, UINT32_MAX if not seen. */
2033	uint32_t idxLastCheckIrqCallNo;
2034	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
2035	uint32_t fSkippingEFlags;
2036	#endif
2037	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
2038	struct
2039	{
2040	/** EFLAGS status bits that we're currently postponing the calculcation of. */
2041	uint32_t fEFlags;
2042	/** The postponed EFLAGS status bits calculation operation. */
2043	IEMNATIVE_POSTPONED_EFL_OP_T enmOp;
2044	/** The bit-width of the postponed EFLAGS calculation. */
2045	uint8_t cOpBits;
2046	/** Host register holding result or first source for the delayed operation,
2047	* UINT8_MAX if not in use. */
2048	uint8_t idxReg1;
2049	/** Host register holding second source for the delayed operation,
2050	* UINT8_MAX if not in use. */
2051	uint8_t idxReg2;
2052	# if defined(VBOX_WITH_STATISTICS) \|\| defined(IEMNATIVE_WITH_TB_DEBUG_INFO)
2053	/** Number of times the delayed calculation was emitted. */
2054	uint8_t cEmits;
2055	# endif
2056	} PostponedEfl;
2057	#endif
2058
2059	/** Core state requiring care with branches. */
2060	IEMNATIVECORESTATE Core;
2061
2062	/** The condition nesting stack. */
2063	IEMNATIVECOND aCondStack[2];
2064
2065	#ifndef IEM_WITH_THROW_CATCH
2066	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
2067	* for recompilation error handling. */
2068	jmp_buf JmpBuf;
2069	#endif
2070	} IEMRECOMPILERSTATE;
2071	/** Pointer to a native recompiler state. */
2072	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
2073
2074
2075	/** @def IEMNATIVE_TRY_SETJMP
2076	* Wrapper around setjmp / try, hiding all the ugly differences.
2077	*
2078	* @note Use with extreme care as this is a fragile macro.
2079	* @param a_pReNative The native recompile state.
2080	* @param a_rcTarget The variable that should receive the status code in case
2081	* of a longjmp/throw.
2082	*/
2083	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
2084	* Start wrapper for catch / setjmp-else.
2085	*
2086	* This will set up a scope.
2087	*
2088	* @note Use with extreme care as this is a fragile macro.
2089	* @param a_pReNative The native recompile state.
2090	* @param a_rcTarget The variable that should receive the status code in case
2091	* of a longjmp/throw.
2092	*/
2093	/** @def IEMNATIVE_CATCH_LONGJMP_END
2094	* End wrapper for catch / setjmp-else.
2095	*
2096	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
2097	* up the state.
2098	*
2099	* @note Use with extreme care as this is a fragile macro.
2100	* @param a_pReNative The native recompile state.
2101	*/
2102	/** @def IEMNATIVE_DO_LONGJMP
2103	*
2104	* Wrapper around longjmp / throw.
2105	*
2106	* @param a_pReNative The native recompile state.
2107	* @param a_rc The status code jump back with / throw.
2108	*/
2109	#ifdef IEM_WITH_THROW_CATCH
2110	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
2111	a_rcTarget = VINF_SUCCESS; \
2112	try
2113	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
2114	catch (int rcThrown) \
2115	{ \
2116	a_rcTarget = rcThrown
2117	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
2118	} \
2119	((void)0)
2120	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
2121	#else /* !IEM_WITH_THROW_CATCH */
2122	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
2123	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
2124	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
2125	else \
2126	{ \
2127	((void)0)
2128	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
2129	}
2130	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
2131	#endif /* !IEM_WITH_THROW_CATCH */
2132
2133
2134	/**
2135	* Native recompiler worker for a threaded function.
2136	*
2137	* @returns New code buffer offset; throws VBox status code in case of a failure.
2138	* @param pReNative The native recompiler state.
2139	* @param off The current code buffer offset.
2140	* @param pCallEntry The threaded call entry.
2141	*
2142	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
2143	*/
2144	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
2145	/** Pointer to a native recompiler worker for a threaded function. */
2146	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
2147
2148	/** Defines a native recompiler worker for a threaded function.
2149	* @see FNIEMNATIVERECOMPFUNC */
2150	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
2151	IEM_DECL_MSC_GUARD_IGNORE uint32_t VBOXCALL \
2152	a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
2153
2154	/** Prototypes a native recompiler function for a threaded function.
2155	* @see FNIEMNATIVERECOMPFUNC */
2156	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
2157
2158
2159	/**
2160	* Native recompiler liveness analysis worker for a threaded function.
2161	*
2162	* @param pCallEntry The threaded call entry.
2163	* @param pIncoming The incoming liveness state entry.
2164	* @param pOutgoing The outgoing liveness state entry.
2165	*/
2166	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
2167	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
2168	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
2169	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
2170
2171	/** Defines a native recompiler liveness analysis worker for a threaded function.
2172	* @see FNIEMNATIVELIVENESSFUNC */
2173	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
2174	IEM_DECL_MSC_GUARD_IGNORE DECLCALLBACK(void) \
2175	a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
2176
2177	/** Prototypes a native recompiler liveness analysis function for a threaded function.
2178	* @see FNIEMNATIVELIVENESSFUNC */
2179	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
2180
2181
2182	/** Define a native recompiler helper function, safe to call from the TB code. */
2183	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
2184	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
2185	/** Prototype a native recompiler helper function, safe to call from the TB code. */
2186	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
2187	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
2188	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
2189	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
2190	a_RetType (VBOXCALL *a_Name) a_ArgList
2191
2192
2193	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2194	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
2195	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
2196	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
2197	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2198	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
2199	IEMNATIVEGSTSIMDREG enmGstSimdReg,
2200	uint8_t idxHstSimdReg = UINT8_MAX,
2201	uint8_t idxHstSimdRegPrev = UINT8_MAX);
2202	# endif
2203	# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
2204	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
2205	uint8_t idxGstReg, uint8_t idxHstReg);
2206	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
2207	uint64_t fGstReg);
2208	# endif
2209	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
2210	uint64_t offPc, uint32_t cInstrSkipped);
2211	# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
2212	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddPostponedEFlagsCalc(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2213	IEMNATIVE_POSTPONED_EFL_OP_T enmOp, uint8_t cOpBits,
2214	uint8_t idxInstance);
2215	# endif
2216	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
2217
2218	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
2219	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
2220	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
2221	DECLHIDDEN(uint32_t) iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
2222	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
2223	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
2224	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
2225	DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere,
2226	IEMNATIVELABELTYPE enmExitReason);
2227	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
2228
2229	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff);
2230	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpPreferNonVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff);
2231	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask);
2232	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpExPreferNonVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask);
2233	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm);
2234
2235	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegReadOnly(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2236	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegUpdate(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2237	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegFullWrite(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2238	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegCalculation(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2239	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegReadOnlyNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2240	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegUpdateNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2241	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegFullWriteNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2242	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegCalculationNoVolatile(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg);
2243
2244	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && defined(VBOX_STRICT)
2245	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestEFlagsReadOnly(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2246	uint64_t fRead, uint64_t fWrite = 0, uint64_t fPotentialCall = 0);
2247	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestEFlagsForUpdate(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2248	uint64_t fRead, uint64_t fWrite = 0, uint64_t fPotentialCall = 0);
2249	#endif
2250
2251	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2252	IEMNATIVEGSTREG enmGstReg);
2253	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) && defined(VBOX_STRICT)
2254	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestEFlagsIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2255	uint64_t fRead, uint64_t fWrite = 0);
2256	#else
2257	DECL_FORCE_INLINE_THROW(uint8_t)
2258	iemNativeRegAllocTmpForGuestEFlagsIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2259	uint64_t fRead, uint64_t fWrite = 0)
2260	{
2261	RT_NOREF(fRead, fWrite);
2262	return iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(pReNative, poff, kIemNativeGstReg_EFlags);
2263	}
2264	#endif
2265
2266	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
2267	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
2268	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
2269	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
2270	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
2271	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2272	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
2273	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
2274	# endif
2275	#endif
2276	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
2277	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
2278	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
2279	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
2280	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2281	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
2282	# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2283	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
2284	# endif
2285	#endif
2286	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
2287	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
2288	uint32_t fKeepVars = 0);
2289	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
2290	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
2291	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2292	uint32_t fHstRegsActiveShadows);
2293	#ifdef VBOX_STRICT
2294	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
2295	#endif
2296	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
2297	uint64_t fGstSimdShwExcept);
2298	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2299	# ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING_DEBUG
2300	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off);
2301	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcDebugCheckWithReg(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxPcReg);
2302	# endif
2303	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
2304	#endif
2305	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2306	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
2307	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWriteEx(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2308	PIEMNATIVECORESTATE pCore, IEMNATIVEGSTREG enmGstReg);
2309	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2310	uint64_t fFlushGstReg = UINT64_MAX);
2311	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative,
2312	uint32_t off, uint8_t idxHstReg);
2313	#endif
2314
2315
2316	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2317	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
2318	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
2319	bool fPreferVolatile = true);
2320	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
2321	IEMNATIVEGSTSIMDREG enmGstSimdReg,
2322	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
2323	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
2324	bool fNoVolatileRegs = false);
2325	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
2326	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
2327	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2328	IEMNATIVEGSTSIMDREG enmGstSimdReg);
2329	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2330	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
2331	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
2332	#endif
2333
2334	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
2335	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
2336	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
2337	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
2338	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
2339	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
2340	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
2341	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
2342	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
2343	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
2344	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
2345	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
2346	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
2347	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2348	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
2349	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
2350	#endif
2351	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
2352	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
2353	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2354	uint32_t fHstRegsNotToSave);
2355	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2356	uint32_t fHstRegsNotToSave);
2357	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
2358	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
2359
2360	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2361	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
2362	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowRegEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off,
2363	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
2364	#ifdef VBOX_STRICT
2365	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
2366	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
2367	IEMNATIVEGSTREG enmGstReg);
2368	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheckEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf,
2369	uint32_t off, uint8_t idxReg, IEMNATIVEGSTREG enmGstReg);
2370	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2371	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
2372	IEMNATIVEGSTSIMDREG enmGstSimdReg,
2373	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
2374	# endif
2375	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
2376	#endif
2377	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
2378	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
2379	#endif
2380	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
2381	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
2382	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
2383	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
2384	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
2385	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
2386	PCIEMTHRDEDCALLENTRY pCallEntry);
2387	IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(iemNativeLivenessFunc_ThreadedCall);
2388	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
2389	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
2390
2391
2392	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
2393	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
2394	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
2395	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
2396	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
2397	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
2398	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
2399	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
2400	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
2401	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
2402	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
2403	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
2404	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
2405
2406	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2407	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2408	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2409	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2410	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2411	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2412	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2413	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2414	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2415	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
2416	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2417	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
2418	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
2419	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
2420	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
2421	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
2422	#endif
2423	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
2424	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
2425	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
2426	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
2427	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2428	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
2429	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
2430	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
2431	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
2432	#endif
2433	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2434	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2435	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2436	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2437	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2438	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2439	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2440
2441	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2442	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2443	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2444	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2445	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2446	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2447	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2448	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2449	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2450	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2451	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2452	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
2453	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
2454	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
2455	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
2456	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
2457	#endif
2458	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
2459	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2460	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2461	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2462	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2463	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
2464	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
2465	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
2466	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
2467	#endif
2468	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
2469	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2470	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
2471	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
2472	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2473	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2474	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
2475
2476	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2477	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2478	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2479	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2480	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2481	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2482	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2483	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2484	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2485	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2486	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2487	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2488	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2489	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2490	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2491	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2492	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2493	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2494	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2495	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2496	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2497	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
2498
2499	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2500	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2501	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2502	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2503	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2504	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2505	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2506	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2507	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2508	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2509	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2510	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2511	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2512	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2513	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2514	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2515	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2516	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2517	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2518	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2519	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2520	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
2521
2522	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2523	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2524	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2525	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
2526
2527
2528	/**
2529	* Info about shadowed guest register values.
2530	* @see IEMNATIVEGSTREG
2531	*/
2532	typedef struct IEMANTIVEGSTREGINFO
2533	{
2534	/** Offset in VMCPU. */
2535	uint32_t off;
2536	/** The field size. */
2537	uint8_t cb;
2538	/** Name (for logging). */
2539	const char *pszName;
2540	} IEMANTIVEGSTREGINFO;
2541	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
2542	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
2543	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
2544	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
2545	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
2546
2547
2548
2549	/**
2550	* Ensures that there is sufficient space in the instruction output buffer.
2551	*
2552	* This will reallocate the buffer if needed and allowed.
2553	*
2554	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
2555	* allocation size.
2556	*
2557	* @returns Pointer to the instruction output buffer on success; throws VBox
2558	* status code on failure, so no need to check it.
2559	* @param pReNative The native recompile state.
2560	* @param off Current instruction offset. Works safely for UINT32_MAX
2561	* as well.
2562	* @param cInstrReq Number of instruction about to be added. It's okay to
2563	* overestimate this a bit.
2564	*/
2565	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
2566	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
2567	{
2568	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
2569	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
2570	{
2571	#ifdef VBOX_STRICT
2572	pReNative->offInstrBufChecked = offChecked;
2573	#endif
2574	return pReNative->pInstrBuf;
2575	}
2576	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
2577	}
2578
2579	/**
2580	* Checks that we didn't exceed the space requested in the last
2581	* iemNativeInstrBufEnsure() call.
2582	*/
2583	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
2584	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
2585	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
2586
2587	/**
2588	* Checks that a variable index is valid.
2589	*/
2590	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2591	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2592	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2593	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2594	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
2595	("%s=%#x\n", #a_idxVar, a_idxVar))
2596	#else
2597	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
2598	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2599	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
2600	#endif
2601
2602	/**
2603	* Checks that a variable index is valid and that the variable is assigned the
2604	* correct argument number.
2605	* This also adds a RT_NOREF of a_idxVar.
2606	*/
2607	#ifdef IEMNATIVE_VAR_IDX_MAGIC
2608	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2609	RT_NOREF_PV(a_idxVar); \
2610	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
2611	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2612	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
2613	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
2614	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2615	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
2616	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
2617	a_uArgNo)); \
2618	} while (0)
2619	#else
2620	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2621	RT_NOREF_PV(a_idxVar); \
2622	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2623	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
2624	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
2625	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2626	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
2627	} while (0)
2628	#endif
2629
2630
2631	/**
2632	* Checks that a variable has the expected size.
2633	*/
2634	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2635	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2636	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2637	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar, (a_cbVar)))
2638
2639
2640	/**
2641	* Calculates the stack address of a variable as a [r]BP displacement value.
2642	*/
2643	DECL_FORCE_INLINE(int32_t)
2644	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2645	{
2646	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2647	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2648	}
2649
2650
2651	/**
2652	* Releases the variable's register.
2653	*
2654	* The register must have been previously acquired calling
2655	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2656	* iemNativeVarRegisterSetAndAcquire().
2657	*/
2658	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2659	{
2660	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2661	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2662	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2663	}
2664
2665
2666	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2667	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2668	{
2669	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2670	iemNativeVarRegisterRelease(pReNative, idxVar);
2671	}
2672	#endif
2673
2674
2675	/**
2676	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2677	*
2678	* @returns The flush mask.
2679	* @param fCImpl The IEM_CIMPL_F_XXX flags.
2680	* @param fGstShwFlush The starting flush mask.
2681	*/
2682	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2683	{
2684	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2685	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2686	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2687	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2688	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2689	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2690	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2691	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2692	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2693	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2694	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2695	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
2696	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
2697	return fGstShwFlush;
2698	}
2699
2700
2701	/** Number of hidden arguments for CIMPL calls.
2702	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2703	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2704	# define IEM_CIMPL_HIDDEN_ARGS 3
2705	#else
2706	# define IEM_CIMPL_HIDDEN_ARGS 2
2707	#endif
2708
2709
2710	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2711	/** Number of hidden arguments for SSE_AIMPL calls. */
2712	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2713	/** Number of hidden arguments for AVX_AIMPL calls. */
2714	# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2715	#endif
2716
2717
2718	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2719
2720	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2721	/**
2722	* Helper for iemNativeLivenessGetStateByGstReg.
2723	*
2724	* @returns IEMLIVENESS_STATE_XXX
2725	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2726	* ORed together.
2727	*/
2728	DECL_FORCE_INLINE(uint32_t)
2729	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2730	{
2731	/* INPUT trumps anything else. */
2732	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2733	return IEMLIVENESS_STATE_INPUT;
2734
2735	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2736	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2737	{
2738	/* If not all sub-fields are clobbered they must be considered INPUT. */
2739	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2740	return IEMLIVENESS_STATE_INPUT;
2741	return IEMLIVENESS_STATE_CLOBBERED;
2742	}
2743
2744	/* XCPT_OR_CALL trumps UNUSED. */
2745	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2746	return IEMLIVENESS_STATE_XCPT_OR_CALL;
2747
2748	return IEMLIVENESS_STATE_UNUSED;
2749	}
2750	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2751
2752
2753	DECL_FORCE_INLINE(uint32_t)
2754	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2755	{
2756	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2757	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2758	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2759	# else
2760	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2761	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2762	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2763	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 3) & 8);
2764	# endif
2765	}
2766
2767
2768	DECL_FORCE_INLINE(uint32_t)
2769	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2770	{
2771	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2772	if (enmGstReg == kIemNativeGstReg_EFlags)
2773	{
2774	/* Merge the eflags states to one. */
2775	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2776	uRet = RT_BIT_32(uRet);
2777	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2778	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2779	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2780	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2781	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2782	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2783	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2784	# else
2785	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2786	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2787	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2788	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2789	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2790	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2791	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2792	# endif
2793	}
2794	return uRet;
2795	}
2796
2797	# ifdef VBOX_STRICT
2798
2799	/** For assertions only - caller checks that idxCurCall isn't zero. */
2800	DECL_FORCE_INLINE(uint32_t)
2801	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2802	{
2803	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2804	}
2805
2806
2807	/** For assertions only - caller checks that idxCurCall isn't zero. */
2808	DECL_FORCE_INLINE(uint32_t)
2809	iemNativeLivenessGetPrevStateByGstRegEx(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2810	{
2811	return iemNativeLivenessGetStateByGstRegEx(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2812	}
2813
2814	# endif /* VBOX_STRICT */
2815	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2816
2817
2818	/**
2819	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2820	*/
2821	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2822	{
2823	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2824	return IEM_CIMPL_HIDDEN_ARGS;
2825	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2826	return 1;
2827	return 0;
2828	}
2829
2830
2831	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2832	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2833	{
2834	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2835
2836	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2837	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2838	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2839	return (uint8_t)idxReg;
2840	}
2841
2842
2843
2844	/*********************************************************************************************************************************
2845	* Register Allocator (GPR) *
2846	*********************************************************************************************************************************/
2847
2848	#ifdef RT_ARCH_ARM64
2849	# include <iprt/armv8.h>
2850	#endif
2851
2852
2853	/**
2854	* Marks host register @a idxHstReg as containing a shadow copy of guest
2855	* register @a enmGstReg.
2856	*
2857	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2858	* host register before calling.
2859	*/
2860	DECL_FORCE_INLINE(void)
2861	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2862	{
2863	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2864	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2865	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2866
2867	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2868	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2869	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2870	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2871	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2872	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2873	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2874	#else
2875	RT_NOREF(off);
2876	#endif
2877	}
2878
2879
2880	/**
2881	* Clear any guest register shadow claims from @a idxHstReg.
2882	*
2883	* The register does not need to be shadowing any guest registers.
2884	*/
2885	DECL_FORCE_INLINE(void)
2886	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2887	{
2888	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2889	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2890	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2891	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2892	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2893	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2894	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2895	#endif
2896
2897	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2898	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2899	if (fGstRegs)
2900	{
2901	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2902	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2903	while (fGstRegs)
2904	{
2905	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2906	fGstRegs &= ~RT_BIT_64(iGstReg);
2907	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2908	}
2909	}
2910	#else
2911	RT_NOREF(off);
2912	#endif
2913
2914	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2915	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2916	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2917	}
2918
2919
2920	/**
2921	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2922	* and global overview flags.
2923	*/
2924	DECL_FORCE_INLINE(void)
2925	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2926	{
2927	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2928	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2929	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2930	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2931	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2932	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2933	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2934	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2935	Assert(!(pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(enmGstReg)));
2936	#endif
2937
2938	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2939	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2940	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2941	#else
2942	RT_NOREF(off);
2943	#endif
2944
2945	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2946	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2947	if (!fGstRegShadowsNew)
2948	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2949	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2950	}
2951
2952
2953	#if 0 /* unused */
2954	/**
2955	* Clear any guest register shadow claim for @a enmGstReg.
2956	*/
2957	DECL_FORCE_INLINE(void)
2958	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2959	{
2960	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2961	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2962	{
2963	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2964	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2965	}
2966	}
2967	#endif
2968
2969
2970	/**
2971	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2972	* as the new shadow of it.
2973	*
2974	* Unlike the other guest reg shadow helpers, this does the logging for you.
2975	* However, it is the liveness state is not asserted here, the caller must do
2976	* that.
2977	*/
2978	DECL_FORCE_INLINE(void)
2979	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2980	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2981	{
2982	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2983	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2984	{
2985	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2986	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2987	if (idxHstRegOld == idxHstRegNew)
2988	return;
2989	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2990	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2991	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2992	}
2993	else
2994	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2995	g_aGstShadowInfo[enmGstReg].pszName));
2996	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2997	}
2998
2999
3000	/**
3001	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
3002	* to @a idxRegTo.
3003	*/
3004	DECL_FORCE_INLINE(void)
3005	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
3006	IEMNATIVEGSTREG enmGstReg, uint32_t off)
3007	{
3008	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
3009	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
3010	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
3011	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
3012	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
3013	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
3014	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
3015	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
3016	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
3017
3018	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
3019	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
3020	if (!fGstRegShadowsFrom)
3021	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
3022	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
3023	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
3024	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
3025	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3026	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3027	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
3028	#else
3029	RT_NOREF(off);
3030	#endif
3031	}
3032
3033
3034	/**
3035	* Flushes any delayed guest register writes.
3036	*
3037	* This must be called prior to calling CImpl functions and any helpers that use
3038	* the guest state (like raising exceptions) and such.
3039	*
3040	* This optimization has not yet been implemented. The first target would be
3041	* RIP updates, since these are the most common ones.
3042	*
3043	* @note This function does not flush any shadowing information for guest
3044	* registers. This needs to be done by the caller if it wishes to do so.
3045	*/
3046	DECL_INLINE_THROW(uint32_t)
3047	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0,
3048	uint64_t fGstSimdShwExcept = 0)
3049	{
3050	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
3051	uint64_t const fWritebackPc = ~fGstShwExcept & RT_BIT_64(kIemNativeGstReg_Pc);
3052	#else
3053	uint64_t const fWritebackPc = 0;
3054	#endif
3055	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
3056	uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
3057	#else
3058	uint64_t const bmGstRegShadowDirty = 0;
3059	#endif
3060	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
3061	uint64_t const bmGstSimdRegShadowDirty = ( pReNative->Core.bmGstSimdRegShadowDirtyLo128
3062	\| pReNative->Core.bmGstSimdRegShadowDirtyHi128)
3063	& ~fGstSimdShwExcept;
3064	#else
3065	uint64_t const bmGstSimdRegShadowDirty = 0;
3066	#endif
3067	if (bmGstRegShadowDirty \| bmGstSimdRegShadowDirty \| fWritebackPc)
3068	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
3069
3070	return off;
3071	}
3072
3073
3074	/**
3075	* Allocates a temporary host general purpose register for keeping a guest
3076	* register value.
3077	*
3078	* Since we may already have a register holding the guest register value,
3079	* code will be emitted to do the loading if that's not the case. Code may also
3080	* be emitted if we have to free up a register to satify the request.
3081	*
3082	* @returns The host register number; throws VBox status code on failure, so no
3083	* need to check the return value.
3084	* @param pReNative The native recompile state.
3085	* @param poff Pointer to the variable with the code buffer
3086	* position. This will be update if we need to move a
3087	* variable from register to stack in order to satisfy
3088	* the request.
3089	* @param enmGstReg The guest register that will is to be updated.
3090	* @param enmIntendedUse How the caller will be using the host register.
3091	* @param fNoVolatileRegs Set if no volatile register allowed, clear if any
3092	* register is okay (default). The ASSUMPTION here is
3093	* that the caller has already flushed all volatile
3094	* registers, so this is only applied if we allocate a
3095	* new register.
3096	* @sa iemNativeRegAllocTmpForGuestEFlags
3097	* iemNativeRegAllocTmpForGuestRegIfAlreadyPresent
3098	* iemNativeRegAllocTmpForGuestRegInt
3099	*/
3100	DECL_FORCE_INLINE_THROW(uint8_t)
3101	iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg,
3102	IEMNATIVEGSTREGUSE const enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
3103	bool const fNoVolatileRegs = false)
3104	{
3105	if (enmIntendedUse == kIemNativeGstRegUse_ReadOnly)
3106	return !fNoVolatileRegs
3107	? iemNativeRegAllocTmpForGuestRegReadOnly(pReNative, poff, enmGstReg)
3108	: iemNativeRegAllocTmpForGuestRegReadOnlyNoVolatile(pReNative, poff, enmGstReg);
3109	if (enmIntendedUse == kIemNativeGstRegUse_ForUpdate)
3110	return !fNoVolatileRegs
3111	? iemNativeRegAllocTmpForGuestRegUpdate(pReNative, poff, enmGstReg)
3112	: iemNativeRegAllocTmpForGuestRegUpdateNoVolatile(pReNative, poff, enmGstReg);
3113	if (enmIntendedUse == kIemNativeGstRegUse_ForFullWrite)
3114	return !fNoVolatileRegs
3115	? iemNativeRegAllocTmpForGuestRegFullWrite(pReNative, poff, enmGstReg)
3116	: iemNativeRegAllocTmpForGuestRegFullWriteNoVolatile(pReNative, poff, enmGstReg);
3117	Assert(enmIntendedUse == kIemNativeGstRegUse_Calculation);
3118	return !fNoVolatileRegs
3119	? iemNativeRegAllocTmpForGuestRegCalculation(pReNative, poff, enmGstReg)
3120	: iemNativeRegAllocTmpForGuestRegCalculationNoVolatile(pReNative, poff, enmGstReg);
3121	}
3122
3123	#if !defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| !defined(VBOX_STRICT)
3124
3125	DECL_FORCE_INLINE_THROW(uint8_t)
3126	iemNativeRegAllocTmpForGuestEFlagsReadOnly(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t fRead,
3127	uint64_t fWrite = 0, uint64_t fPotentialCall = 0)
3128	{
3129	RT_NOREF(fRead, fWrite, fPotentialCall);
3130	return iemNativeRegAllocTmpForGuestRegReadOnly(pReNative, poff, kIemNativeGstReg_EFlags);
3131	}
3132
3133	DECL_FORCE_INLINE_THROW(uint8_t)
3134	iemNativeRegAllocTmpForGuestEFlagsForUpdate(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t fRead,
3135	uint64_t fWrite = 0, uint64_t fPotentialCall = 0)
3136	{
3137	RT_NOREF(fRead, fWrite, fPotentialCall);
3138	return iemNativeRegAllocTmpForGuestRegUpdate(pReNative, poff, kIemNativeGstReg_EFlags);
3139	}
3140
3141	#endif
3142
3143
3144
3145	/*********************************************************************************************************************************
3146	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
3147	*********************************************************************************************************************************/
3148
3149	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
3150
3151	DECL_FORCE_INLINE(uint8_t)
3152	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
3153	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
3154	{
3155	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
3156
3157	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
3158	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
3159	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
3160	return idxSimdReg;
3161	}
3162
3163
3164	/**
3165	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
3166	* SIMD register @a enmGstSimdReg.
3167	*
3168	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
3169	* host register before calling.
3170	*/
3171	DECL_FORCE_INLINE(void)
3172	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
3173	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
3174	{
3175	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
3176	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
3177	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
3178
3179	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
3180	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
3181	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
3182	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
3183	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3184	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3185	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
3186	#else
3187	RT_NOREF(off);
3188	#endif
3189	}
3190
3191
3192	/**
3193	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
3194	* to @a idxSimdRegTo.
3195	*/
3196	DECL_FORCE_INLINE(void)
3197	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
3198	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
3199	{
3200	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
3201	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
3202	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
3203	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
3204	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
3205	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
3206	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
3207	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
3208	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
3209	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
3210	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
3211
3212	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
3213	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
3214	if (!fGstRegShadowsFrom)
3215	{
3216	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
3217	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
3218	}
3219	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
3220	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
3221	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
3222	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3223	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3224	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
3225	#else
3226	RT_NOREF(off);
3227	#endif
3228	}
3229
3230
3231	/**
3232	* Clear any guest register shadow claims from @a idxHstSimdReg.
3233	*
3234	* The register does not need to be shadowing any guest registers.
3235	*/
3236	DECL_FORCE_INLINE(void)
3237	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
3238	{
3239	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
3240	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
3241	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
3242	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
3243	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
3244	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
3245	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
3246
3247	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
3248	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
3249	if (fGstRegs)
3250	{
3251	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
3252	iemNativeDbgInfoAddNativeOffset(pReNative, off);
3253	while (fGstRegs)
3254	{
3255	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
3256	fGstRegs &= ~RT_BIT_64(iGstReg);
3257	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
3258	}
3259	}
3260	#else
3261	RT_NOREF(off);
3262	#endif
3263
3264	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
3265	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
3266	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
3267	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
3268	}
3269
3270	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
3271
3272
3273	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
3274	/**
3275	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
3276	*/
3277	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
3278	{
3279	if (pReNative->Core.offPc)
3280	return iemNativeEmitPcWritebackSlow(pReNative, off);
3281	return off;
3282	}
3283	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
3284
3285
3286	/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
3287	* it saves us the trouble of a hidden parameter on MSC/amd64. */
3288	#ifdef RT_ARCH_AMD64
3289	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
3290	#elif defined(RT_ARCH_ARM64)
3291	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
3292	#endif
3293
3294	#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
3295	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeFpCtrlRegRestore, (uint64_t u64RegFpCtrl));
3296	#endif
3297
3298	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
3299
3300	/** @} */
3301
3302	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
3303

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 106401

Download in other formats: