IEMN8veRecompiler.h@ 105490

Last change on this file since 105490 was 105490, checked in by vboxsync, 7 months ago

VMM/IEM: Basic infrastructure to natively recompile SIMD floating point instructions, bugref:10652

SIMD floating point operation behavior depends on the guest MXCSR value which needs to be written to the
host's floating point control register (MXCSR on x86, FPCR on arm64 which needs conversion) and needs to be
restored to the host's value when the TB finished execution to avoid inconsistencies in case the guest
changes MXCSR. The ARM implementation does not conform to the x86 behavior because default NaN values have
the sign bit clear on arm64 while they are set on x86. There are rounding differences as well and earlier
ARMv8 revisions don't support the FPCR.FIZ and FPCR.AH features. Should still work out as long as the guest
doesn't try to do funny stuff.

Property svn:eol-style set to native
Property svn:keywords set to Author Date Id Revision

File size: 128.9 KB

Line
1	/* $Id: IEMN8veRecompiler.h 105490 2024-07-24 14:49:29Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42	/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43	* Enables generating internal debug info for better TB disassembly dumping. */
44	#if defined(DEBUG) \|\| defined(DOXYGEN_RUNNING)
45	# define IEMNATIVE_WITH_TB_DEBUG_INFO
46	#endif
47
48	/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49	* Enables liveness analysis. */
50	#if 1 \|\| defined(DOXYGEN_RUNNING)
51	# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52	/# define IEMLIVENESS_EXTENDED_LAYOUT/
53	#endif
54
55	/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
56	* Enables skipping EFLAGS calculations/updating based on liveness info. */
57	#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) \|\| defined(DOXYGEN_RUNNING)
58	# define IEMNATIVE_WITH_EFLAGS_SKIPPING
59	#endif
60
61	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
62	* Enables strict consistency checks around EFLAGS skipping.
63	* @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
64	#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
65	# ifdef VBOX_STRICT
66	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
67	# endif
68	#elif defined(DOXYGEN_RUNNING)
69	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
70	#endif
71
72	#ifdef VBOX_WITH_STATISTICS
73	/** Always count instructions for now. */
74	# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
75	#endif
76
77	/** @def IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
78	* Enables having only a single prologue for native TBs. */
79	#if 1 \|\| defined(DOXYGEN_RUNNING)
80	# define IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
81	#endif
82
83	/** @def IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
84	* Enable this to use common epilogue and tail code for all TBs in a chunk. */
85	#if 1 \|\| defined(DOXYGEN_RUNNING)
86	# define IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
87	#endif
88
89
90	/** @name Stack Frame Layout
91	*
92	* @{ */
93	/** The size of the area for stack variables and spills and stuff.
94	* @note This limit is duplicated in the python script(s). We add 0x40 for
95	* alignment padding. */
96	#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
97	/** Number of 64-bit variable slots (0x100 / 8 = 32. */
98	#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
99	AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
100
101	#ifdef RT_ARCH_AMD64
102	/** An stack alignment adjustment (between non-volatile register pushes and
103	* the stack variable area, so the latter better aligned). */
104	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
105
106	/** Number of stack arguments slots for calls made from the frame. */
107	# ifdef RT_OS_WINDOWS
108	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
109	# else
110	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
111	# endif
112	/** Number of any shadow arguments (spill area) for calls we make. */
113	# ifdef RT_OS_WINDOWS
114	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
115	# else
116	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
117	# endif
118
119	/** Frame pointer (RBP) relative offset of the last push. */
120	# ifdef RT_OS_WINDOWS
121	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
122	# else
123	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
124	# endif
125	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
126	* address for it). */
127	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
128	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
129	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
130	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
131	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
132	# ifdef RT_OS_WINDOWS
133	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
134	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
135	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
136	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
137	# endif
138
139	# ifdef RT_OS_WINDOWS
140	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
141	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
142	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
143	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
144	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
145	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
146	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
147	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
148	# endif
149
150	#elif RT_ARCH_ARM64
151	/** No alignment padding needed for arm64. */
152	# define IEMNATIVE_FRAME_ALIGN_SIZE 0
153	/** No stack argument slots, got 8 registers for arguments will suffice. */
154	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
155	/** There are no argument spill area. */
156	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
157
158	/** Number of saved registers at the top of our stack frame.
159	* This includes the return address and old frame pointer, so x19 thru x30. */
160	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
161	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
162	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
163
164	/** Frame pointer (BP) relative offset of the last push. */
165	# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
166
167	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
168	* address for it). */
169	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
170
171	#else
172	# error "port me"
173	#endif
174	/** @} */
175
176
177	/** @name Fixed Register Allocation(s)
178	* @{ */
179	/** @def IEMNATIVE_REG_FIXED_PVMCPU
180	* The number of the register holding the pVCpu pointer. */
181	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
182	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
183	* @note This not available on AMD64, only ARM64. */
184	/** @def IEMNATIVE_REG_FIXED_TMP0
185	* Dedicated temporary register.
186	* @todo replace this by a register allocator and content tracker. */
187	/** @def IEMNATIVE_REG_FIXED_MASK
188	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
189	* architecture. */
190	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
191	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
192	* Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
193	* architecture. */
194	/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
195	* Dedicated temporary SIMD register. */
196	#endif
197	#if defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
198	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
199	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
200	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
201	# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
202	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
203	# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
204	# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
205	# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
206	# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
207	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
208	# else
209	# define IEMNATIVE_REG_FIXED_MASK_ADD 0
210	# endif
211	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
212	\| RT_BIT_32(ARMV8_A64_REG_LR) \
213	\| RT_BIT_32(ARMV8_A64_REG_BP) \
214	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
215	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
216	\| RT_BIT_32(ARMV8_A64_REG_X18) \
217	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
218	\| IEMNATIVE_REG_FIXED_MASK_ADD)
219
220	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
221	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
222	# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
223	# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
224	# else
225	/** @note
226	* ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
227	* support emulating 256-bit registers we pair two real registers statically to
228	* one virtual for now, leaving us with only 16 256-bit registers. We always
229	* pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
230	* the register allocator assumes that it will be always free when the lower is
231	* picked.
232	*
233	* Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
234	* touch them in order to avoid having to save and restore them in the
235	* prologue/epilogue.
236	*/
237	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
238	\| RT_BIT_32(ARMV8_A64_REG_Q31) \
239	\| RT_BIT_32(ARMV8_A64_REG_Q30) \
240	\| RT_BIT_32(ARMV8_A64_REG_Q29) \
241	\| RT_BIT_32(ARMV8_A64_REG_Q27) \
242	\| RT_BIT_32(ARMV8_A64_REG_Q25) \
243	\| RT_BIT_32(ARMV8_A64_REG_Q23) \
244	\| RT_BIT_32(ARMV8_A64_REG_Q21) \
245	\| RT_BIT_32(ARMV8_A64_REG_Q19) \
246	\| RT_BIT_32(ARMV8_A64_REG_Q17) \
247	\| RT_BIT_32(ARMV8_A64_REG_Q15) \
248	\| RT_BIT_32(ARMV8_A64_REG_Q13) \
249	\| RT_BIT_32(ARMV8_A64_REG_Q11) \
250	\| RT_BIT_32(ARMV8_A64_REG_Q9) \
251	\| RT_BIT_32(ARMV8_A64_REG_Q7) \
252	\| RT_BIT_32(ARMV8_A64_REG_Q5) \
253	\| RT_BIT_32(ARMV8_A64_REG_Q3) \
254	\| RT_BIT_32(ARMV8_A64_REG_Q1))
255	# endif
256	# endif
257
258	#elif defined(RT_ARCH_AMD64)
259	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
260	# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
261	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
262	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
263	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
264	\| RT_BIT_32(X86_GREG_xSP) \
265	\| RT_BIT_32(X86_GREG_xBP) )
266
267	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
268	# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
269	# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
270	# ifndef _MSC_VER
271	# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
272	# endif
273	# endif
274	# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
275	# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
276	# else
277	/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
278	# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
279	\| RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
280	# endif
281	# endif
282
283	#else
284	# error "port me"
285	#endif
286	/** @} */
287
288	/** @name Call related registers.
289	* @{ */
290	/** @def IEMNATIVE_CALL_RET_GREG
291	* The return value register. */
292	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
293	* Number of arguments in registers. */
294	/** @def IEMNATIVE_CALL_ARG0_GREG
295	* The general purpose register carrying argument \#0. */
296	/** @def IEMNATIVE_CALL_ARG1_GREG
297	* The general purpose register carrying argument \#1. */
298	/** @def IEMNATIVE_CALL_ARG2_GREG
299	* The general purpose register carrying argument \#2. */
300	/** @def IEMNATIVE_CALL_ARG3_GREG
301	* The general purpose register carrying argument \#3. */
302	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
303	* Mask of registers the callee will not save and may trash. */
304	#ifdef RT_ARCH_AMD64
305	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
306
307	# ifdef RT_OS_WINDOWS
308	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
309	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
310	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
311	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
312	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
313	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
314	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
315	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
316	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
317	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
318	\| RT_BIT_32(X86_GREG_xCX) \
319	\| RT_BIT_32(X86_GREG_xDX) \
320	\| RT_BIT_32(X86_GREG_x8) \
321	\| RT_BIT_32(X86_GREG_x9) \
322	\| RT_BIT_32(X86_GREG_x10) \
323	\| RT_BIT_32(X86_GREG_x11) )
324	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
325	/* xmm0 - xmm5 are marked as volatile. */
326	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
327	# endif
328
329	# else /* !RT_OS_WINDOWS */
330	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
331	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
332	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
333	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
334	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
335	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
336	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
337	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
338	\| RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
339	\| RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
340	\| RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
341	\| RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
342	\| RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
343	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
344	\| RT_BIT_32(X86_GREG_xCX) \
345	\| RT_BIT_32(X86_GREG_xDX) \
346	\| RT_BIT_32(X86_GREG_xDI) \
347	\| RT_BIT_32(X86_GREG_xSI) \
348	\| RT_BIT_32(X86_GREG_x8) \
349	\| RT_BIT_32(X86_GREG_x9) \
350	\| RT_BIT_32(X86_GREG_x10) \
351	\| RT_BIT_32(X86_GREG_x11) )
352	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
353	/* xmm0 - xmm15 are marked as volatile. */
354	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
355	# endif
356	# endif /* !RT_OS_WINDOWS */
357
358	#elif defined(RT_ARCH_ARM64)
359	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
360	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
361	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
362	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
363	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
364	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
365	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
366	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
367	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
368	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
369	# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
370	\| RT_BIT_32(ARMV8_A64_REG_X1) \
371	\| RT_BIT_32(ARMV8_A64_REG_X2) \
372	\| RT_BIT_32(ARMV8_A64_REG_X3) \
373	\| RT_BIT_32(ARMV8_A64_REG_X4) \
374	\| RT_BIT_32(ARMV8_A64_REG_X5) \
375	\| RT_BIT_32(ARMV8_A64_REG_X6) \
376	\| RT_BIT_32(ARMV8_A64_REG_X7) )
377	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
378	\| RT_BIT_32(ARMV8_A64_REG_X1) \
379	\| RT_BIT_32(ARMV8_A64_REG_X2) \
380	\| RT_BIT_32(ARMV8_A64_REG_X3) \
381	\| RT_BIT_32(ARMV8_A64_REG_X4) \
382	\| RT_BIT_32(ARMV8_A64_REG_X5) \
383	\| RT_BIT_32(ARMV8_A64_REG_X6) \
384	\| RT_BIT_32(ARMV8_A64_REG_X7) \
385	\| RT_BIT_32(ARMV8_A64_REG_X8) \
386	\| RT_BIT_32(ARMV8_A64_REG_X9) \
387	\| RT_BIT_32(ARMV8_A64_REG_X10) \
388	\| RT_BIT_32(ARMV8_A64_REG_X11) \
389	\| RT_BIT_32(ARMV8_A64_REG_X12) \
390	\| RT_BIT_32(ARMV8_A64_REG_X13) \
391	\| RT_BIT_32(ARMV8_A64_REG_X14) \
392	\| RT_BIT_32(ARMV8_A64_REG_X15) \
393	\| RT_BIT_32(ARMV8_A64_REG_X16) \
394	\| RT_BIT_32(ARMV8_A64_REG_X17) )
395	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
396	/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
397	* so to simplify our life a bit we just mark everything as volatile. */
398	# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
399	# endif
400
401	#endif
402
403	/** This is the maximum argument count we'll ever be needing. */
404	#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
405	#ifdef RT_OS_WINDOWS
406	# ifdef VBOXSTRICTRC_STRICT_ENABLED
407	# undef IEMNATIVE_CALL_MAX_ARG_COUNT
408	# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
409	# endif
410	#endif
411	/** @} */
412
413
414	/** @def IEMNATIVE_HST_GREG_COUNT
415	* Number of host general purpose registers we tracker. */
416	/** @def IEMNATIVE_HST_GREG_MASK
417	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
418	* inverted register masks and such to get down to a correct set of regs. */
419	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
420	/** @def IEMNATIVE_HST_SIMD_REG_COUNT
421	* Number of host SIMD registers we track. */
422	/** @def IEMNATIVE_HST_SIMD_REG_MASK
423	* Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
424	* inverted register masks and such to get down to a correct set of regs. */
425	#endif
426	#ifdef RT_ARCH_AMD64
427	# define IEMNATIVE_HST_GREG_COUNT 16
428	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
429
430	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
431	# define IEMNATIVE_HST_SIMD_REG_COUNT 16
432	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
433	# endif
434
435	#elif defined(RT_ARCH_ARM64)
436	# define IEMNATIVE_HST_GREG_COUNT 32
437	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
438
439	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
440	# define IEMNATIVE_HST_SIMD_REG_COUNT 32
441	# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
442	# endif
443
444	#else
445	# error "Port me!"
446	#endif
447
448
449	#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
450
451
452	/** Native code generator label types. */
453	typedef enum
454	{
455	kIemNativeLabelType_Invalid = 0,
456	/*
457	* Labels w/o data, only once instance per TB - aka exit reasons.
458	*
459	* Note! Jumps to these requires instructions that are capable of spanning
460	* the max TB length.
461	*/
462	/* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
463	kIemNativeLabelType_RaiseDe, /*< Raise (throw) X86_XCPT_DE (00h). /
464	kIemNativeLabelType_RaiseUd, /*< Raise (throw) X86_XCPT_UD (06h). /
465	kIemNativeLabelType_RaiseSseRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. /
466	kIemNativeLabelType_RaiseAvxRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. /
467	kIemNativeLabelType_RaiseSseAvxFpRelated, /*< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. /
468	kIemNativeLabelType_RaiseNm, /*< Raise (throw) X86_XCPT_NM (07h). /
469	kIemNativeLabelType_RaiseGp0, /*< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. /
470	kIemNativeLabelType_RaiseMf, /*< Raise (throw) X86_XCPT_MF (10h). /
471	kIemNativeLabelType_RaiseXf, /*< Raise (throw) X86_XCPT_XF (13h). /
472	kIemNativeLabelType_ObsoleteTb,
473	kIemNativeLabelType_NeedCsLimChecking,
474	kIemNativeLabelType_CheckBranchMiss,
475	kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
476	/* Manually defined labels. */
477	kIemNativeLabelType_ReturnBreak,
478	kIemNativeLabelType_ReturnBreakFF,
479	kIemNativeLabelType_ReturnBreakViaLookup,
480	kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
481	kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
482	kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
483	kIemNativeLabelType_ReturnWithFlags,
484	kIemNativeLabelType_NonZeroRetOrPassUp,
485	kIemNativeLabelType_Return,
486	/** The last fixup for branches that can span almost the whole TB length.
487	* @note Whether kIemNativeLabelType_Return needs to be one of these is
488	* a bit questionable, since nobody jumps to it except other tail code. */
489	kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_Return,
490	/** The last fixup for branches that exits the TB. */
491	kIemNativeLabelType_LastTbExit = kIemNativeLabelType_Return,
492
493	/*
494	* Labels with data, potentially multiple instances per TB:
495	*
496	* These are localized labels, so no fixed jump type restrictions here.
497	*/
498	kIemNativeLabelType_FirstWithMultipleInstances,
499	kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
500	kIemNativeLabelType_Else,
501	kIemNativeLabelType_Endif,
502	kIemNativeLabelType_CheckIrq,
503	kIemNativeLabelType_TlbLookup,
504	kIemNativeLabelType_TlbMiss,
505	kIemNativeLabelType_TlbDone,
506	kIemNativeLabelType_End
507	} IEMNATIVELABELTYPE;
508
509	#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
510	((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
511
512
513	/** Native code generator label definition. */
514	typedef struct IEMNATIVELABEL
515	{
516	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
517	* the epilog. */
518	uint32_t off;
519	/** The type of label (IEMNATIVELABELTYPE). */
520	uint16_t enmType;
521	/** Additional label data, type specific. */
522	uint16_t uData;
523	} IEMNATIVELABEL;
524	/** Pointer to a label. */
525	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
526
527
528	/** Native code generator fixup types. */
529	typedef enum
530	{
531	kIemNativeFixupType_Invalid = 0,
532	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
533	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
534	kIemNativeFixupType_Rel32,
535	#elif defined(RT_ARCH_ARM64)
536	/** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
537	kIemNativeFixupType_RelImm26At0,
538	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
539	kIemNativeFixupType_RelImm19At5,
540	/** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
541	kIemNativeFixupType_RelImm14At5,
542	#endif
543	kIemNativeFixupType_End
544	} IEMNATIVEFIXUPTYPE;
545
546	/** Native code generator fixup. */
547	typedef struct IEMNATIVEFIXUP
548	{
549	/** Code offset of the fixup location. */
550	uint32_t off;
551	/** The IEMNATIVELABEL this is a fixup for. */
552	uint16_t idxLabel;
553	/** The fixup type (IEMNATIVEFIXUPTYPE). */
554	uint8_t enmType;
555	/** Addend or other data. */
556	int8_t offAddend;
557	} IEMNATIVEFIXUP;
558	/** Pointer to a native code generator fixup. */
559	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
560
561	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
562
563	/** Native code generator fixup to per chunk TB tail code. */
564	typedef struct IEMNATIVEEXITFIXUP
565	{
566	/** Code offset of the fixup location. */
567	uint32_t off;
568	/** The exit reason. */
569	IEMNATIVELABELTYPE enmExitReason;
570	} IEMNATIVEEXITFIXUP;
571	/** Pointer to a native code generator TB exit fixup. */
572	typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
573
574	/**
575	* Per executable memory chunk context with addresses for common code.
576	*/
577	typedef struct IEMNATIVEPERCHUNKCTX
578	{
579	/** Pointers to the exit labels */
580	PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
581	} IEMNATIVEPERCHUNKCTX;
582	/** Pointer to per-chunk recompiler context. */
583	typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
584	/** Pointer to const per-chunk recompiler context. */
585	typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
586
587	#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
588
589
590	/**
591	* One bit of the state.
592	*
593	* Each register state takes up two bits. We keep the two bits in two separate
594	* 64-bit words to simplify applying them to the guest shadow register mask in
595	* the register allocator.
596	*/
597	typedef union IEMLIVENESSBIT
598	{
599	uint64_t bm64;
600	RT_GCC_EXTENSION struct
601	{ /* bit no */
602	uint64_t bmGprs : 16; /*< 0x00 / 0: The 16 general purpose registers. /
603	uint64_t fUnusedPc : 1; /*< 0x10 / 16: (PC in ) /
604	uint64_t fCr0 : 1; /*< 0x11 / 17: /
605	uint64_t fFcw : 1; /*< 0x12 / 18: /
606	uint64_t fFsw : 1; /*< 0x13 / 19: /
607	uint64_t bmSegBase : 6; /*< 0x14 / 20: /
608	uint64_t bmSegAttrib : 6; /*< 0x1a / 26: /
609	uint64_t bmSegLimit : 6; /*< 0x20 / 32: /
610	uint64_t bmSegSel : 6; /*< 0x26 / 38: /
611	uint64_t fCr4 : 1; /*< 0x2c / 44: /
612	uint64_t fXcr0 : 1; /*< 0x2d / 45: /
613	uint64_t fMxCsr : 1; /*< 0x2e / 46: /
614	uint64_t fEflOther : 1; /*< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! /
615	uint64_t fEflCf : 1; /*< 0x30 / 48: Carry flag (X86_EFL_CF / 0). /
616	uint64_t fEflPf : 1; /*< 0x31 / 49: Parity flag (X86_EFL_PF / 2). /
617	uint64_t fEflAf : 1; /*< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). /
618	uint64_t fEflZf : 1; /*< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). /
619	uint64_t fEflSf : 1; /*< 0x34 / 52: Signed flag (X86_EFL_SF / 7). /
620	uint64_t fEflOf : 1; /*< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). /
621	uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
622	};
623	} IEMLIVENESSBIT;
624	AssertCompileSize(IEMLIVENESSBIT, 8);
625
626	#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
627	#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
628	#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
629	#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
630	#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
631	#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
632	#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
633
634
635	/**
636	* A liveness state entry.
637	*
638	* The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
639	* Once we add a SSE register shadowing, we'll add another 64-bit element for
640	* that.
641	*/
642	typedef union IEMLIVENESSENTRY
643	{
644	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
645	uint64_t bm64[16 / 8];
646	uint16_t bm32[16 / 4];
647	uint16_t bm16[16 / 2];
648	uint8_t bm8[ 16 / 1];
649	IEMLIVENESSBIT aBits[2];
650	#else
651	uint64_t bm64[32 / 8];
652	uint16_t bm32[32 / 4];
653	uint16_t bm16[32 / 2];
654	uint8_t bm8[ 32 / 1];
655	IEMLIVENESSBIT aBits[4];
656	#endif
657	RT_GCC_EXTENSION struct
658	{
659	/** Bit \#0 of the register states. */
660	IEMLIVENESSBIT Bit0;
661	/** Bit \#1 of the register states. */
662	IEMLIVENESSBIT Bit1;
663	#ifdef IEMLIVENESS_EXTENDED_LAYOUT
664	/** Bit \#2 of the register states. */
665	IEMLIVENESSBIT Bit2;
666	/** Bit \#3 of the register states. */
667	IEMLIVENESSBIT Bit3;
668	#endif
669	};
670	} IEMLIVENESSENTRY;
671	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
672	AssertCompileSize(IEMLIVENESSENTRY, 16);
673	#else
674	AssertCompileSize(IEMLIVENESSENTRY, 32);
675	#endif
676	/** Pointer to a liveness state entry. */
677	typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
678	/** Pointer to a const liveness state entry. */
679	typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
680
681	/** @name 64-bit value masks for IEMLIVENESSENTRY.
682	* @{ / / 0xzzzzyyyyxxxxwwww */
683	#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
684
685	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
686	# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
687	# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
688
689	# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
690	# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
691	#endif
692
693	#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
694	#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
695
696	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
697	# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
698	# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
699	#endif
700	/** @} */
701
702
703	/** @name The liveness state for a register.
704	*
705	* The state values have been picked to with state accumulation in mind (what
706	* the iemNativeLivenessFunc_xxxx functions does), as that is the most
707	* performance critical work done with the values.
708	*
709	* This is a compressed state that only requires 2 bits per register.
710	* When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
711	* 1. the incoming state from the following call,
712	* 2. the outgoing state for this call,
713	* 3. mask of the entries set in the 2nd.
714	*
715	* The mask entry (3rd one above) will be used both when updating the outgoing
716	* state and when merging in incoming state for registers not touched by the
717	* current call.
718	*
719	* @{ */
720	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
721	/** The register will be clobbered and the current value thrown away.
722	*
723	* When this is applied to the state (2) we'll simply be AND'ing it with the
724	* (old) mask (3) and adding the register to the mask. This way we'll
725	* preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
726	* IEMLIVENESS_STATE_INPUT states. */
727	# define IEMLIVENESS_STATE_CLOBBERED 0
728	/** The register is unused in the remainder of the TB.
729	*
730	* This is an initial state and can not be set by any of the
731	* iemNativeLivenessFunc_xxxx callbacks. */
732	# define IEMLIVENESS_STATE_UNUSED 1
733	/** The register value is required in a potential call or exception.
734	*
735	* This means that the register value must be calculated and is best written to
736	* the state, but that any shadowing registers can be flushed thereafter as it's
737	* not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
738	*
739	* It is typically applied across the board, but we preserve incoming
740	* IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
741	* trickery to filter out IEMLIVENESS_STATE_UNUSED:
742	* 1. r0 = old & ~mask;
743	* 2. r0 = t1 & (t1 >> 1)'
744	* 3. state \|= r0 \| 0b10;
745	* 4. mask = ~0;
746	*/
747	# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
748	/** The register value is used as input.
749	*
750	* This means that the register value must be calculated and it is best to keep
751	* it in a register. It does not need to be writtent out as such. This is the
752	* highest priority state.
753	*
754	* Whether the call modifies the register or not isn't relevant to earlier
755	* calls, so that's not recorded.
756	*
757	* When applying this state we just or in the value in the outgoing state and
758	* mask. */
759	# define IEMLIVENESS_STATE_INPUT 3
760	/** Mask of the state bits. */
761	# define IEMLIVENESS_STATE_MASK 3
762	/** The number of bits per state. */
763	# define IEMLIVENESS_STATE_BIT_COUNT 2
764	/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
765	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
766	/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
767	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
768	/** Check if a register clobbering is expected given the (previous) liveness state.
769	* The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
770	* include INPUT if the register is used in more than one place. */
771	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
772
773	/** Check if all status flags are going to be clobbered and doesn't need
774	* calculating in the current step.
775	* @param a_pCurEntry The current liveness entry. */
776	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
777	( (((a_pCurEntry)->Bit0.bm64 \| (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
778
779	#else /* IEMLIVENESS_EXTENDED_LAYOUT */
780	/** The register is not used any more. */
781	# define IEMLIVENESS_STATE_UNUSED 0
782	/** Flag: The register is required in a potential exception or call. */
783	# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
784	# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
785	/** Flag: The register is read. */
786	# define IEMLIVENESS_STATE_READ 2
787	# define IEMLIVENESS_BIT_READ 1
788	/** Flag: The register is written. */
789	# define IEMLIVENESS_STATE_WRITE 4
790	# define IEMLIVENESS_BIT_WRITE 2
791	/** Flag: Unconditional call (not needed, can be redefined for research). */
792	# define IEMLIVENESS_STATE_CALL 8
793	# define IEMLIVENESS_BIT_CALL 3
794	# define IEMLIVENESS_BIT_OTHER 3 /*< More convenient name for this one. /
795	# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
796	( ((a_uState) & (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE \| IEMLIVENESS_STATE_READ) )
797	# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
798	# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
799
800	# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
801	( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
802	&& !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \| (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
803	& IEMLIVENESSBIT_STATUS_EFL_MASK) )
804
805	#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
806	/** @} */
807
808	/** @name Liveness helpers for builtin functions and similar.
809	*
810	* These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
811	* own set of manimulator macros for those.
812	*
813	* @{ */
814	/** Initializing the state as all unused. */
815	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
816	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
817	do { \
818	(a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
819	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
820	} while (0)
821	#else
822	# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
823	do { \
824	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
825	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
826	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
827	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
828	} while (0)
829	#endif
830
831	/** Initializing the outgoing state with a potential xcpt or call state.
832	* This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
833	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
834	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
835	do { \
836	(a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
837	(a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
838	} while (0)
839	#else
840	# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
841	do { \
842	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
843	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
844	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
845	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
846	} while (0)
847	#endif
848
849	/** Adds a segment base register as input to the outgoing state. */
850	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
851	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
852	(a_pOutgoing)->Bit0.bmSegBase \|= RT_BIT_64(a_iSReg); \
853	(a_pOutgoing)->Bit1.bmSegBase \|= RT_BIT_64(a_iSReg); \
854	} while (0)
855	#else
856	# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
857	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase \|= RT_BIT_64(a_iSReg); \
858	} while (0)
859	#endif
860
861	/** Adds a segment attribute register as input to the outgoing state. */
862	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
863	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
864	(a_pOutgoing)->Bit0.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
865	(a_pOutgoing)->Bit1.bmSegAttrib \|= RT_BIT_64(a_iSReg); \
866	} while (0)
867	#else
868	# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
869	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib \|= RT_BIT_64(a_iSReg); \
870	} while (0)
871	#endif
872
873	/** Adds a segment limit register as input to the outgoing state. */
874	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
875	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
876	(a_pOutgoing)->Bit0.bmSegLimit \|= RT_BIT_64(a_iSReg); \
877	(a_pOutgoing)->Bit1.bmSegLimit \|= RT_BIT_64(a_iSReg); \
878	} while (0)
879	#else
880	# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
881	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit \|= RT_BIT_64(a_iSReg); \
882	} while (0)
883	#endif
884
885	/** Adds a segment limit register as input to the outgoing state. */
886	#ifndef IEMLIVENESS_EXTENDED_LAYOUT
887	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
888	(a_pOutgoing)->Bit0.a_fEflMember \|= 1; \
889	(a_pOutgoing)->Bit1.a_fEflMember \|= 1; \
890	} while (0)
891	#else
892	# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
893	(a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember \|= 1; \
894	} while (0)
895	#endif
896	/** @} */
897
898	/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
899	* Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
900	* calculated and up to date. This is to double check that we haven't skipped
901	* EFLAGS calculations when we actually need them. NOP in non-strict builds.
902	* @note has to be placed in
903	*/
904	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
905	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
906	do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
907	#else
908	# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
909	#endif
910
911
912	/**
913	* Guest registers that can be shadowed in GPRs.
914	*
915	* This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
916	* must be placed last, as the liveness state tracks it as 7 subcomponents and
917	* we don't want to waste space here.
918	*
919	* @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
920	* friends as well as IEMAllN8veLiveness.cpp.
921	*/
922	typedef enum IEMNATIVEGSTREG : uint8_t
923	{
924	kIemNativeGstReg_GprFirst = 0,
925	kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
926	kIemNativeGstReg_Pc,
927	kIemNativeGstReg_Cr0,
928	kIemNativeGstReg_FpuFcw,
929	kIemNativeGstReg_FpuFsw,
930	kIemNativeGstReg_SegBaseFirst,
931	kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
932	kIemNativeGstReg_SegAttribFirst,
933	kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
934	kIemNativeGstReg_SegLimitFirst,
935	kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
936	kIemNativeGstReg_SegSelFirst,
937	kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
938	kIemNativeGstReg_Cr4,
939	kIemNativeGstReg_Xcr0,
940	kIemNativeGstReg_MxCsr,
941	kIemNativeGstReg_EFlags, /*< 32-bit, includes internal flags - last! /
942	kIemNativeGstReg_End
943	} IEMNATIVEGSTREG;
944	AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
945	AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
946
947	/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
948	* @{ */
949	#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
950	#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
951	#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
952	#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
953	#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
954	/** @} */
955
956	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
957
958	/**
959	* Guest registers that can be shadowed in host SIMD registers.
960	*
961	* @todo r=aeichner Liveness tracking
962	* @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
963	*/
964	typedef enum IEMNATIVEGSTSIMDREG : uint8_t
965	{
966	kIemNativeGstSimdReg_SimdRegFirst = 0,
967	kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
968	kIemNativeGstSimdReg_End
969	} IEMNATIVEGSTSIMDREG;
970
971	/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
972	* @{ */
973	#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
974	/** @} */
975
976	/**
977	* The Load/store size for a SIMD guest register.
978	*/
979	typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
980	{
981	/** Invalid size. */
982	kIemNativeGstSimdRegLdStSz_Invalid = 0,
983	/** Loads the low 128-bit of a guest SIMD register. */
984	kIemNativeGstSimdRegLdStSz_Low128,
985	/** Loads the high 128-bit of a guest SIMD register. */
986	kIemNativeGstSimdRegLdStSz_High128,
987	/** Loads the whole 256-bits of a guest SIMD register. */
988	kIemNativeGstSimdRegLdStSz_256,
989	/** End value. */
990	kIemNativeGstSimdRegLdStSz_End
991	} IEMNATIVEGSTSIMDREGLDSTSZ;
992
993	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
994
995	/**
996	* Intended use statement for iemNativeRegAllocTmpForGuestReg().
997	*/
998	typedef enum IEMNATIVEGSTREGUSE
999	{
1000	/** The usage is read-only, the register holding the guest register
1001	* shadow copy will not be modified by the caller. */
1002	kIemNativeGstRegUse_ReadOnly = 0,
1003	/** The caller will update the guest register (think: PC += cbInstr).
1004	* The guest shadow copy will follow the returned register. */
1005	kIemNativeGstRegUse_ForUpdate,
1006	/** The call will put an entirely new value in the guest register, so
1007	* if new register is allocate it will be returned uninitialized. */
1008	kIemNativeGstRegUse_ForFullWrite,
1009	/** The caller will use the guest register value as input in a calculation
1010	* and the host register will be modified.
1011	* This means that the returned host register will not be marked as a shadow
1012	* copy of the guest register. */
1013	kIemNativeGstRegUse_Calculation
1014	} IEMNATIVEGSTREGUSE;
1015
1016	/**
1017	* Guest registers (classes) that can be referenced.
1018	*/
1019	typedef enum IEMNATIVEGSTREGREF : uint8_t
1020	{
1021	kIemNativeGstRegRef_Invalid = 0,
1022	kIemNativeGstRegRef_Gpr,
1023	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
1024	kIemNativeGstRegRef_EFlags,
1025	kIemNativeGstRegRef_MxCsr,
1026	kIemNativeGstRegRef_FpuReg,
1027	kIemNativeGstRegRef_MReg,
1028	kIemNativeGstRegRef_XReg,
1029	kIemNativeGstRegRef_X87,
1030	kIemNativeGstRegRef_XState,
1031	//kIemNativeGstRegRef_YReg, - doesn't work.
1032	kIemNativeGstRegRef_End
1033	} IEMNATIVEGSTREGREF;
1034
1035
1036	/** Variable kinds. */
1037	typedef enum IEMNATIVEVARKIND : uint8_t
1038	{
1039	/** Customary invalid zero value. */
1040	kIemNativeVarKind_Invalid = 0,
1041	/** This is either in a register or on the stack. */
1042	kIemNativeVarKind_Stack,
1043	/** Immediate value - loaded into register when needed, or can live on the
1044	* stack if referenced (in theory). */
1045	kIemNativeVarKind_Immediate,
1046	/** Variable reference - loaded into register when needed, never stack. */
1047	kIemNativeVarKind_VarRef,
1048	/** Guest register reference - loaded into register when needed, never stack. */
1049	kIemNativeVarKind_GstRegRef,
1050	/** End of valid values. */
1051	kIemNativeVarKind_End
1052	} IEMNATIVEVARKIND;
1053
1054
1055	/** Variable or argument. */
1056	typedef struct IEMNATIVEVAR
1057	{
1058	/** The kind of variable. */
1059	IEMNATIVEVARKIND enmKind;
1060	/** The variable size in bytes. */
1061	uint8_t cbVar;
1062	/** The first stack slot (uint64_t), except for immediate and references
1063	* where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1064	* has a stack slot it has been initialized and has a value. Unused variables
1065	* has neither a stack slot nor a host register assignment. */
1066	uint8_t idxStackSlot;
1067	/** The host register allocated for the variable, UINT8_MAX if not. */
1068	uint8_t idxReg;
1069	/** The argument number if argument, UINT8_MAX if regular variable. */
1070	uint8_t uArgNo;
1071	/** If referenced, the index (unpacked) of the variable referencing this one,
1072	* otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1073	* and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1074	uint8_t idxReferrerVar;
1075	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1076	* @todo not sure what this really is for... */
1077	IEMNATIVEGSTREG enmGstReg;
1078	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1079	/** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1080	* only valid when idxReg is not UINT8_MAX. */
1081	bool fSimdReg : 1;
1082	/** Set if the registered is currently used exclusively, false if the
1083	* variable is idle and the register can be grabbed. */
1084	bool fRegAcquired : 1;
1085	#else
1086	/** Set if the registered is currently used exclusively, false if the
1087	* variable is idle and the register can be grabbed. */
1088	bool fRegAcquired;
1089	#endif
1090
1091	union
1092	{
1093	/** kIemNativeVarKind_Immediate: The immediate value. */
1094	uint64_t uValue;
1095	/** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1096	uint8_t idxRefVar;
1097	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1098	struct
1099	{
1100	/** The class of register. */
1101	IEMNATIVEGSTREGREF enmClass;
1102	/** Index within the class. */
1103	uint8_t idx;
1104	} GstRegRef;
1105	} u;
1106	} IEMNATIVEVAR;
1107	/** Pointer to a variable or argument. */
1108	typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1109	/** Pointer to a const variable or argument. */
1110	typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1111
1112	/** What is being kept in a host register. */
1113	typedef enum IEMNATIVEWHAT : uint8_t
1114	{
1115	/** The traditional invalid zero value. */
1116	kIemNativeWhat_Invalid = 0,
1117	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1118	kIemNativeWhat_Var,
1119	/** Temporary register, this is typically freed when a MC completes. */
1120	kIemNativeWhat_Tmp,
1121	/** Call argument w/o a variable mapping. This is free (via
1122	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1123	kIemNativeWhat_Arg,
1124	/** Return status code.
1125	* @todo not sure if we need this... */
1126	kIemNativeWhat_rc,
1127	/** The fixed pVCpu (PVMCPUCC) register.
1128	* @todo consider offsetting this on amd64 to use negative offsets to access
1129	* more members using 8-byte disp. */
1130	kIemNativeWhat_pVCpuFixed,
1131	/** The fixed pCtx (PCPUMCTX) register.
1132	* @todo consider offsetting this on amd64 to use negative offsets to access
1133	* more members using 8-byte disp. */
1134	kIemNativeWhat_pCtxFixed,
1135	/** Fixed temporary register. */
1136	kIemNativeWhat_FixedTmp,
1137	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1138	/** Shadow RIP for the delayed RIP updating debugging. */
1139	kIemNativeWhat_PcShadow,
1140	#endif
1141	/** Register reserved by the CPU or OS architecture. */
1142	kIemNativeWhat_FixedReserved,
1143	/** End of valid values. */
1144	kIemNativeWhat_End
1145	} IEMNATIVEWHAT;
1146
1147	/**
1148	* Host general register entry.
1149	*
1150	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1151	*
1152	* @todo Track immediate values in host registers similarlly to how we track the
1153	* guest register shadow copies. For it to be real helpful, though,
1154	* we probably need to know which will be reused and put them into
1155	* non-volatile registers, otherwise it's going to be more or less
1156	* restricted to an instruction or two.
1157	*/
1158	typedef struct IEMNATIVEHSTREG
1159	{
1160	/** Set of guest registers this one shadows.
1161	*
1162	* Using a bitmap here so we can designate the same host register as a copy
1163	* for more than one guest register. This is expected to be useful in
1164	* situations where one value is copied to several registers in a sequence.
1165	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1166	* sequence we'd want to let this register follow to be a copy of and there
1167	* will always be places where we'd be picking the wrong one.
1168	*/
1169	uint64_t fGstRegShadows;
1170	/** What is being kept in this register. */
1171	IEMNATIVEWHAT enmWhat;
1172	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1173	uint8_t idxVar;
1174	/** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1175	* by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1176	* that scope. */
1177	uint8_t idxStackSlot;
1178	/** Alignment padding. */
1179	uint8_t abAlign[5];
1180	} IEMNATIVEHSTREG;
1181
1182
1183	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1184	/**
1185	* Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1186	* halves, on architectures where there is no 256-bit register available this entry will track
1187	* two adjacent 128-bit host registers.
1188	*
1189	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1190	*/
1191	typedef struct IEMNATIVEHSTSIMDREG
1192	{
1193	/** Set of guest registers this one shadows.
1194	*
1195	* Using a bitmap here so we can designate the same host register as a copy
1196	* for more than one guest register. This is expected to be useful in
1197	* situations where one value is copied to several registers in a sequence.
1198	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1199	* sequence we'd want to let this register follow to be a copy of and there
1200	* will always be places where we'd be picking the wrong one.
1201	*/
1202	uint64_t fGstRegShadows;
1203	/** What is being kept in this register. */
1204	IEMNATIVEWHAT enmWhat;
1205	/** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1206	uint8_t idxVar;
1207	/** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1208	IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1209	/** Alignment padding. */
1210	uint8_t abAlign[5];
1211	} IEMNATIVEHSTSIMDREG;
1212	#endif
1213
1214
1215	/**
1216	* Core state for the native recompiler, that is, things that needs careful
1217	* handling when dealing with branches.
1218	*/
1219	typedef struct IEMNATIVECORESTATE
1220	{
1221	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1222	/** The current instruction offset in bytes from when the guest program counter
1223	* was updated last. Used for delaying the write to the guest context program counter
1224	* as long as possible. */
1225	uint32_t offPc;
1226	/** Number of instructions where we could skip the updating. */
1227	uint32_t cInstrPcUpdateSkipped;
1228	#endif
1229	/** Allocation bitmap for aHstRegs. */
1230	uint32_t bmHstRegs;
1231
1232	/** Bitmap marking which host register contains guest register shadow copies.
1233	* This is used during register allocation to try preserve copies. */
1234	uint32_t bmHstRegsWithGstShadow;
1235	/** Bitmap marking valid entries in aidxGstRegShadows. */
1236	uint64_t bmGstRegShadows;
1237	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1238	/** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1239	uint64_t bmGstRegShadowDirty;
1240	#endif
1241
1242	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1243	/** Allocation bitmap for aHstSimdRegs. */
1244	uint32_t bmHstSimdRegs;
1245
1246	/** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1247	* This is used during register allocation to try preserve copies. */
1248	uint32_t bmHstSimdRegsWithGstShadow;
1249	/** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1250	uint64_t bmGstSimdRegShadows;
1251	/** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1252	uint64_t bmGstSimdRegShadowDirtyLo128;
1253	/** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1254	uint64_t bmGstSimdRegShadowDirtyHi128;
1255	#endif
1256
1257	union
1258	{
1259	/** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1260	uint8_t aidxArgVars[8];
1261	/** For more efficient resetting. */
1262	uint64_t u64ArgVars;
1263	};
1264
1265	/** Allocation bitmap for the stack. */
1266	uint32_t bmStack;
1267	/** Allocation bitmap for aVars. */
1268	uint32_t bmVars;
1269
1270	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1271	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1272	* (A shadow copy of a guest register can only be held in a one host register,
1273	* there are no duplicate copies or ambiguities like that). */
1274	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1275	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1276	/** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1277	* Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1278	* (A shadow copy of a guest register can only be held in a one host register,
1279	* there are no duplicate copies or ambiguities like that). */
1280	uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1281	#endif
1282
1283	/** Host register allocation tracking. */
1284	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1285	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1286	/** Host SIMD register allocation tracking. */
1287	IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1288	#endif
1289
1290	/** Variables and arguments. */
1291	IEMNATIVEVAR aVars[9];
1292	} IEMNATIVECORESTATE;
1293	/** Pointer to core state. */
1294	typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1295	/** Pointer to const core state. */
1296	typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1297
1298	/** @def IEMNATIVE_VAR_IDX_UNPACK
1299	* @returns Index into IEMNATIVECORESTATE::aVars.
1300	* @param a_idxVar Variable index w/ magic (in strict builds).
1301	*/
1302	/** @def IEMNATIVE_VAR_IDX_PACK
1303	* @returns Variable index w/ magic (in strict builds).
1304	* @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1305	*/
1306	#ifdef VBOX_STRICT
1307	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1308	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) \| IEMNATIVE_VAR_IDX_MAGIC)
1309	# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1310	# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1311	# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1312	#else
1313	# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1314	# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1315	#endif
1316
1317
1318	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1319	/** Clear the dirty state of the given guest SIMD register. */
1320	# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1321	do { \
1322	(a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1323	(a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1324	} while (0)
1325
1326	/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1327	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1328	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1329	/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1330	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1331	RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1332	/** Returns whether the given guest SIMD register is dirty. */
1333	# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1334	RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \| (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1335
1336	/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1337	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1338	((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 \|= RT_BIT_64(a_iSimdReg))
1339	/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1340	# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1341	((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 \|= RT_BIT_64(a_iSimdReg))
1342
1343	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1344	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1345	/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1346	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1347	/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1348	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1349	/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1350	# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1351	# ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1352	/** Flag indicating that the guest MXCSR was synced to the host floating point control register. */
1353	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED RT_BIT_32(4)
1354	/** Flag indicating whether the host floating point control register was saved before overwriting it. */
1355	# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SAVED RT_BIT_32(5)
1356	# endif
1357	#endif
1358
1359
1360	/**
1361	* Conditional stack entry.
1362	*/
1363	typedef struct IEMNATIVECOND
1364	{
1365	/** Set if we're in the "else" part, clear if we're in the "if" before it. */
1366	bool fInElse;
1367	/** The label for the IEM_MC_ELSE. */
1368	uint32_t idxLabelElse;
1369	/** The label for the IEM_MC_ENDIF. */
1370	uint32_t idxLabelEndIf;
1371	/** The initial state snapshot as the if-block starts executing. */
1372	IEMNATIVECORESTATE InitialState;
1373	/** The state snapshot at the end of the if-block. */
1374	IEMNATIVECORESTATE IfFinalState;
1375	} IEMNATIVECOND;
1376	/** Pointer to a condition stack entry. */
1377	typedef IEMNATIVECOND *PIEMNATIVECOND;
1378
1379
1380	/**
1381	* Native recompiler state.
1382	*/
1383	typedef struct IEMRECOMPILERSTATE
1384	{
1385	/** Size of the buffer that pbNativeRecompileBufR3 points to in
1386	* IEMNATIVEINSTR units. */
1387	uint32_t cInstrBufAlloc;
1388	#ifdef VBOX_STRICT
1389	/** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1390	uint32_t offInstrBufChecked;
1391	#else
1392	uint32_t uPadding1; /* We don't keep track of the size here... */
1393	#endif
1394	/** Fixed temporary code buffer for native recompilation. */
1395	PIEMNATIVEINSTR pInstrBuf;
1396
1397	/** Bitmaps with the label types used. */
1398	uint64_t bmLabelTypes;
1399	/** Actual number of labels in paLabels. */
1400	uint32_t cLabels;
1401	/** Max number of entries allowed in paLabels before reallocating it. */
1402	uint32_t cLabelsAlloc;
1403	/** Labels defined while recompiling (referenced by fixups). */
1404	PIEMNATIVELABEL paLabels;
1405	/** Array with indexes of unique labels (uData always 0). */
1406	uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1407
1408	/** Actual number of fixups paFixups. */
1409	uint32_t cFixups;
1410	/** Max number of entries allowed in paFixups before reallocating it. */
1411	uint32_t cFixupsAlloc;
1412	/** Buffer used by the recompiler for recording fixups when generating code. */
1413	PIEMNATIVEFIXUP paFixups;
1414
1415	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1416	/** Actual number of fixups in paTbExitFixups. */
1417	uint32_t cTbExitFixups;
1418	/** Max number of entries allowed in paTbExitFixups before reallocating it. */
1419	uint32_t cTbExitFixupsAlloc;
1420	/** Buffer used by the recompiler for recording fixups when generating code. */
1421	PIEMNATIVEEXITFIXUP paTbExitFixups;
1422	#endif
1423
1424	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1425	/** Number of debug info entries allocated for pDbgInfo. */
1426	uint32_t cDbgInfoAlloc;
1427	uint32_t uPadding;
1428	/** Debug info. */
1429	PIEMTBDBG pDbgInfo;
1430	#endif
1431
1432	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1433	/** The current call index (liveness array and threaded calls in TB). */
1434	uint32_t idxCurCall;
1435	/** Number of liveness entries allocated. */
1436	uint32_t cLivenessEntriesAlloc;
1437	/** Liveness entries for all the calls in the TB begin recompiled.
1438	* The entry for idxCurCall contains the info for what the next call will
1439	* require wrt registers. (Which means the last entry is the initial liveness
1440	* state.) */
1441	PIEMLIVENESSENTRY paLivenessEntries;
1442	#endif
1443
1444	/** The translation block being recompiled. */
1445	PCIEMTB pTbOrg;
1446	/** The VMCPU structure of the EMT. */
1447	PVMCPUCC pVCpu;
1448
1449	/** Condition sequence number (for generating unique labels). */
1450	uint16_t uCondSeqNo;
1451	/** Check IRQ seqeunce number (for generating unique labels). */
1452	uint16_t uCheckIrqSeqNo;
1453	/** TLB load sequence number (for generating unique labels). */
1454	uint16_t uTlbSeqNo;
1455	/** The current condition stack depth (aCondStack). */
1456	uint8_t cCondDepth;
1457
1458	/** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1459	uint8_t cArgsX;
1460	/** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1461	uint32_t fCImpl;
1462	/** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1463	uint32_t fMc;
1464	/** The expected IEMCPU::fExec value for the current call/instruction. */
1465	uint32_t fExec;
1466	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1467	/** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1468	* we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1469	*
1470	* This is an optimization because these control registers can only be changed from
1471	* by calling a C helper we can catch. Should reduce the number of instructions in a TB
1472	* consisting of multiple SIMD instructions.
1473	*/
1474	uint32_t fSimdRaiseXcptChecksEmitted;
1475	#endif
1476	/** The call number of the last CheckIrq, UINT32_MAX if not seen. */
1477	uint32_t idxLastCheckIrqCallNo;
1478
1479	/** Core state requiring care with branches. */
1480	IEMNATIVECORESTATE Core;
1481
1482	/** The condition nesting stack. */
1483	IEMNATIVECOND aCondStack[2];
1484
1485	#ifndef IEM_WITH_THROW_CATCH
1486	/** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1487	* for recompilation error handling. */
1488	jmp_buf JmpBuf;
1489	#endif
1490	} IEMRECOMPILERSTATE;
1491	/** Pointer to a native recompiler state. */
1492	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1493
1494
1495	/** @def IEMNATIVE_TRY_SETJMP
1496	* Wrapper around setjmp / try, hiding all the ugly differences.
1497	*
1498	* @note Use with extreme care as this is a fragile macro.
1499	* @param a_pReNative The native recompile state.
1500	* @param a_rcTarget The variable that should receive the status code in case
1501	* of a longjmp/throw.
1502	*/
1503	/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1504	* Start wrapper for catch / setjmp-else.
1505	*
1506	* This will set up a scope.
1507	*
1508	* @note Use with extreme care as this is a fragile macro.
1509	* @param a_pReNative The native recompile state.
1510	* @param a_rcTarget The variable that should receive the status code in case
1511	* of a longjmp/throw.
1512	*/
1513	/** @def IEMNATIVE_CATCH_LONGJMP_END
1514	* End wrapper for catch / setjmp-else.
1515	*
1516	* This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1517	* up the state.
1518	*
1519	* @note Use with extreme care as this is a fragile macro.
1520	* @param a_pReNative The native recompile state.
1521	*/
1522	/** @def IEMNATIVE_DO_LONGJMP
1523	*
1524	* Wrapper around longjmp / throw.
1525	*
1526	* @param a_pReNative The native recompile state.
1527	* @param a_rc The status code jump back with / throw.
1528	*/
1529	#ifdef IEM_WITH_THROW_CATCH
1530	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1531	a_rcTarget = VINF_SUCCESS; \
1532	try
1533	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1534	catch (int rcThrown) \
1535	{ \
1536	a_rcTarget = rcThrown
1537	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1538	} \
1539	((void)0)
1540	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1541	#else /* !IEM_WITH_THROW_CATCH */
1542	# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1543	if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1544	# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1545	else \
1546	{ \
1547	((void)0)
1548	# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1549	}
1550	# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1551	#endif /* !IEM_WITH_THROW_CATCH */
1552
1553
1554	/**
1555	* Native recompiler worker for a threaded function.
1556	*
1557	* @returns New code buffer offset; throws VBox status code in case of a failure.
1558	* @param pReNative The native recompiler state.
1559	* @param off The current code buffer offset.
1560	* @param pCallEntry The threaded call entry.
1561	*
1562	* @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1563	*/
1564	typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1565	/** Pointer to a native recompiler worker for a threaded function. */
1566	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1567
1568	/** Defines a native recompiler worker for a threaded function.
1569	* @see FNIEMNATIVERECOMPFUNC */
1570	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1571	uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1572
1573	/** Prototypes a native recompiler function for a threaded function.
1574	* @see FNIEMNATIVERECOMPFUNC */
1575	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1576
1577
1578	/**
1579	* Native recompiler liveness analysis worker for a threaded function.
1580	*
1581	* @param pCallEntry The threaded call entry.
1582	* @param pIncoming The incoming liveness state entry.
1583	* @param pOutgoing The outgoing liveness state entry.
1584	*/
1585	typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1586	PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1587	/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1588	typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1589
1590	/** Defines a native recompiler liveness analysis worker for a threaded function.
1591	* @see FNIEMNATIVELIVENESSFUNC */
1592	#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1593	DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1594
1595	/** Prototypes a native recompiler liveness analysis function for a threaded function.
1596	* @see FNIEMNATIVELIVENESSFUNC */
1597	#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1598
1599
1600	/** Define a native recompiler helper function, safe to call from the TB code. */
1601	#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1602	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1603	/** Prototype a native recompiler helper function, safe to call from the TB code. */
1604	#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1605	DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1606	/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1607	#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1608	a_RetType (VBOXCALL *a_Name) a_ArgList
1609
1610
1611	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1612	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1613	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1614	uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1615	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1616	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1617	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1618	uint8_t idxHstSimdReg = UINT8_MAX,
1619	uint8_t idxHstSimdRegPrev = UINT8_MAX);
1620	# endif
1621	# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) \|\| defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1622	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1623	uint8_t idxGstReg, uint8_t idxHstReg);
1624	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1625	uint64_t fGstReg);
1626	# endif
1627	DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1628	uint32_t offPc, uint32_t cInstrSkipped);
1629	#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1630
1631	DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1632	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1633	DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1634	DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1635	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1636	#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1637	DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVELABELTYPE enmExitReason);
1638	#endif
1639	DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1640
1641	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1642	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1643	bool fPreferVolatile = true);
1644	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1645	bool fPreferVolatile = true);
1646	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1647	IEMNATIVEGSTREG enmGstReg,
1648	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1649	bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1650	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1651	IEMNATIVEGSTREG enmGstReg);
1652
1653	DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1654	DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1655	#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) \|\| (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1656	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1657	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1658	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1659	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1660	uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1661	# endif
1662	#endif
1663	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1664	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1665	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1666	DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1667	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1668	DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1669	# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1670	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1671	# endif
1672	#endif
1673	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1674	DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1675	uint32_t fKeepVars = 0);
1676	DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1677	DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1678	DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1679	uint32_t fHstRegsActiveShadows);
1680	#ifdef VBOX_STRICT
1681	DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1682	#endif
1683	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1684	uint64_t fGstSimdShwExcept);
1685	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1686	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1687	#endif
1688	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1689	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1690	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fFlushGstReg = UINT64_MAX);
1691	DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1692	#endif
1693
1694
1695	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1696	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1697	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1698	bool fPreferVolatile = true);
1699	DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1700	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1701	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1702	IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1703	bool fNoVolatileRegs = false);
1704	DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1705	DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1706	DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1707	IEMNATIVEGSTSIMDREG enmGstSimdReg);
1708	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1709	uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1710	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1711	#endif
1712
1713	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1714	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1715	DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1716	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1717	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1718	DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
1719	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1720	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1721	DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1722	IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1723	DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1724	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1725	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1726	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1727	DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1728	bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1729	#endif
1730	DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1731	IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1732	DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1733	uint32_t fHstRegsNotToSave);
1734	DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1735	uint32_t fHstRegsNotToSave);
1736	DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1737	DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1738
1739	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1740	uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1741	#ifdef VBOX_STRICT
1742	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1743	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1744	IEMNATIVEGSTREG enmGstReg);
1745	# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1746	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1747	IEMNATIVEGSTSIMDREG enmGstSimdReg,
1748	IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1749	# endif
1750	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1751	#endif
1752	#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1753	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1754	#endif
1755	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1756	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
1757	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1758	uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1759	uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1760	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1761	PCIEMTHRDEDCALLENTRY pCallEntry);
1762	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1763	uint8_t idxAddrReg, uint8_t idxInstr);
1764	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1765	uint8_t idxAddrReg, uint8_t idxInstr);
1766	DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1767	IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1768
1769
1770	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1771	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1772	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1773	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1774	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1775	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1776	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1777	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1778	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1779	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1780	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
1781	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
1782	IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
1783
1784	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1785	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1786	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1787	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1788	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1789	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1790	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1791	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1792	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1793	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1794	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1795	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1796	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1797	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1798	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1799	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1800	#endif
1801	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1802	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1803	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1804	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1805	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1806	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1807	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1808	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1809	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1810	#endif
1811	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1812	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1813	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1814	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1815	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1816	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1817	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1818
1819	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1820	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1821	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1822	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1823	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1824	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1825	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1826	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1827	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1828	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1829	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1830	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1831	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1832	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1833	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1834	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1835	#endif
1836	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1837	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1838	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1839	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1840	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1841	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1842	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1843	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1844	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1845	#endif
1846	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1847	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1848	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1849	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1850	IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1851	IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1852	IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1853
1854	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1855	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1856	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1857	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1858	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1859	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1860	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1861	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1862	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1863	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1864	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1865	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1866	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1867	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1868	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1869	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1870	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1871	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1872	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1873	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1874	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1875	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1876
1877	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1878	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1879	IEM_DECL_NATIVE_HLP_PROTO(uint8_t , iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1880	IEM_DECL_NATIVE_HLP_PROTO(uint8_t const , iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1881	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1882	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1883	IEM_DECL_NATIVE_HLP_PROTO(uint16_t , iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1884	IEM_DECL_NATIVE_HLP_PROTO(uint16_t const , iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1885	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1886	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1887	IEM_DECL_NATIVE_HLP_PROTO(uint32_t , iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1888	IEM_DECL_NATIVE_HLP_PROTO(uint32_t const , iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1889	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1890	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1891	IEM_DECL_NATIVE_HLP_PROTO(uint64_t , iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1892	IEM_DECL_NATIVE_HLP_PROTO(uint64_t const , iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1893	IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U , iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1894	IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U , iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1895	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1896	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1897	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U , iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1898	IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const , iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, RTGCPTR GCPtrMem));
1899
1900	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1901	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1902	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1903	IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1904
1905
1906	/**
1907	* Info about shadowed guest register values.
1908	* @see IEMNATIVEGSTREG
1909	*/
1910	typedef struct IEMANTIVEGSTREGINFO
1911	{
1912	/** Offset in VMCPU. */
1913	uint32_t off;
1914	/** The field size. */
1915	uint8_t cb;
1916	/** Name (for logging). */
1917	const char *pszName;
1918	} IEMANTIVEGSTREGINFO;
1919	extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1920	extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1921	extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1922	extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1923	extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1924
1925
1926
1927	/**
1928	* Ensures that there is sufficient space in the instruction output buffer.
1929	*
1930	* This will reallocate the buffer if needed and allowed.
1931	*
1932	* @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1933	* allocation size.
1934	*
1935	* @returns Pointer to the instruction output buffer on success; throws VBox
1936	* status code on failure, so no need to check it.
1937	* @param pReNative The native recompile state.
1938	* @param off Current instruction offset. Works safely for UINT32_MAX
1939	* as well.
1940	* @param cInstrReq Number of instruction about to be added. It's okay to
1941	* overestimate this a bit.
1942	*/
1943	DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1944	iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1945	{
1946	uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1947	if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1948	{
1949	#ifdef VBOX_STRICT
1950	pReNative->offInstrBufChecked = offChecked;
1951	#endif
1952	return pReNative->pInstrBuf;
1953	}
1954	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1955	}
1956
1957	/**
1958	* Checks that we didn't exceed the space requested in the last
1959	* iemNativeInstrBufEnsure() call.
1960	*/
1961	#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1962	AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1963	("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1964
1965	/**
1966	* Checks that a variable index is valid.
1967	*/
1968	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1969	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1970	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1971	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1972	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1973	("%s=%#x\n", #a_idxVar, a_idxVar))
1974	#else
1975	# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1976	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1977	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1978	#endif
1979
1980	/**
1981	* Checks that a variable index is valid and that the variable is assigned the
1982	* correct argument number.
1983	* This also adds a RT_NOREF of a_idxVar.
1984	*/
1985	#ifdef IEMNATIVE_VAR_IDX_MAGIC
1986	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1987	RT_NOREF_PV(a_idxVar); \
1988	AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1989	&& (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1990	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1991	&& (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1992	("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1993	(a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1994	RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
1995	a_uArgNo)); \
1996	} while (0)
1997	#else
1998	# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1999	RT_NOREF_PV(a_idxVar); \
2000	AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2001	&& ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
2002	&& (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
2003	, ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2004	(a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
2005	} while (0)
2006	#endif
2007
2008
2009	/**
2010	* Checks that a variable has the expected size.
2011	*/
2012	#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2013	AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2014	("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2015	(a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar, (a_cbVar)))
2016
2017
2018	/**
2019	* Calculates the stack address of a variable as a [r]BP displacement value.
2020	*/
2021	DECL_FORCE_INLINE(int32_t)
2022	iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2023	{
2024	Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2025	return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2026	}
2027
2028
2029	/**
2030	* Releases the variable's register.
2031	*
2032	* The register must have been previously acquired calling
2033	* iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2034	* iemNativeVarRegisterSetAndAcquire().
2035	*/
2036	DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2037	{
2038	IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2039	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2040	pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2041	}
2042
2043
2044	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2045	DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2046	{
2047	Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2048	iemNativeVarRegisterRelease(pReNative, idxVar);
2049	}
2050	#endif
2051
2052
2053	/**
2054	* Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2055	*
2056	* @returns The flush mask.
2057	* @param fCImpl The IEM_CIMPL_F_XXX flags.
2058	* @param fGstShwFlush The starting flush mask.
2059	*/
2060	DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2061	{
2062	if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2063	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2064	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2065	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2066	if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2067	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2068	\| RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2069	\| RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2070	\| RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2071	else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2072	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2073	if (fCImpl & (IEM_CIMPL_F_RFLAGS \| IEM_CIMPL_F_STATUS_FLAGS \| IEM_CIMPL_F_INHIBIT_SHADOW))
2074	fGstShwFlush \|= RT_BIT_64(kIemNativeGstReg_EFlags);
2075	return fGstShwFlush;
2076	}
2077
2078
2079	/** Number of hidden arguments for CIMPL calls.
2080	* @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2081	#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2082	# define IEM_CIMPL_HIDDEN_ARGS 3
2083	#else
2084	# define IEM_CIMPL_HIDDEN_ARGS 2
2085	#endif
2086
2087
2088	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2089	/** Number of hidden arguments for SSE_AIMPL calls. */
2090	# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2091	/** Number of hidden arguments for AVX_AIMPL calls. */
2092	# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2093	#endif
2094
2095
2096	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2097
2098	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2099	/**
2100	* Helper for iemNativeLivenessGetStateByGstReg.
2101	*
2102	* @returns IEMLIVENESS_STATE_XXX
2103	* @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2104	* ORed together.
2105	*/
2106	DECL_FORCE_INLINE(uint32_t)
2107	iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2108	{
2109	/* INPUT trumps anything else. */
2110	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2111	return IEMLIVENESS_STATE_INPUT;
2112
2113	/* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2114	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2115	{
2116	/* If not all sub-fields are clobbered they must be considered INPUT. */
2117	if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) \| RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2118	return IEMLIVENESS_STATE_INPUT;
2119	return IEMLIVENESS_STATE_CLOBBERED;
2120	}
2121
2122	/* XCPT_OR_CALL trumps UNUSED. */
2123	if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2124	return IEMLIVENESS_STATE_XCPT_OR_CALL;
2125
2126	return IEMLIVENESS_STATE_UNUSED;
2127	}
2128	# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2129
2130
2131	DECL_FORCE_INLINE(uint32_t)
2132	iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2133	{
2134	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2135	return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2136	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2137	# else
2138	return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2139	\| (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2140	\| (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2141	\| (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
2142	# endif
2143	}
2144
2145
2146	DECL_FORCE_INLINE(uint32_t)
2147	iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2148	{
2149	uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2150	if (enmGstReg == kIemNativeGstReg_EFlags)
2151	{
2152	/* Merge the eflags states to one. */
2153	# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2154	uRet = RT_BIT_32(uRet);
2155	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflCf \| (pLivenessEntry->Bit1.fEflCf << 1));
2156	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflPf \| (pLivenessEntry->Bit1.fEflPf << 1));
2157	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflAf \| (pLivenessEntry->Bit1.fEflAf << 1));
2158	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflZf \| (pLivenessEntry->Bit1.fEflZf << 1));
2159	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflSf \| (pLivenessEntry->Bit1.fEflSf << 1));
2160	uRet \|= RT_BIT_32(pLivenessEntry->Bit0.fEflOf \| (pLivenessEntry->Bit1.fEflOf << 1));
2161	uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2162	# else
2163	AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2164	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2165	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2166	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2167	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2168	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2169	uRet \|= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2170	# endif
2171	}
2172	return uRet;
2173	}
2174
2175
2176	# ifdef VBOX_STRICT
2177	/** For assertions only, user checks that idxCurCall isn't zerow. */
2178	DECL_FORCE_INLINE(uint32_t)
2179	iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2180	{
2181	return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2182	}
2183	# endif /* VBOX_STRICT */
2184
2185	#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2186
2187
2188	/**
2189	* Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2190	*/
2191	DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2192	{
2193	if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2194	return IEM_CIMPL_HIDDEN_ARGS;
2195	if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE \| IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2196	return 1;
2197	return 0;
2198	}
2199
2200
2201	DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2202	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2203	{
2204	pReNative->Core.bmHstRegs \|= RT_BIT_32(idxReg);
2205
2206	pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2207	pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2208	pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2209	return (uint8_t)idxReg;
2210	}
2211
2212
2213
2214	/*********************************************************************************************************************************
2215	* Register Allocator (GPR) *
2216	*********************************************************************************************************************************/
2217
2218	/**
2219	* Marks host register @a idxHstReg as containing a shadow copy of guest
2220	* register @a enmGstReg.
2221	*
2222	* ASSUMES that caller has made sure @a enmGstReg is not associated with any
2223	* host register before calling.
2224	*/
2225	DECL_FORCE_INLINE(void)
2226	iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2227	{
2228	Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2229	Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2230	Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2231
2232	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2233	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2234	pReNative->Core.bmGstRegShadows \|= RT_BIT_64(enmGstReg);
2235	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxHstReg);
2236	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2237	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2238	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2239	#else
2240	RT_NOREF(off);
2241	#endif
2242	}
2243
2244
2245	/**
2246	* Clear any guest register shadow claims from @a idxHstReg.
2247	*
2248	* The register does not need to be shadowing any guest registers.
2249	*/
2250	DECL_FORCE_INLINE(void)
2251	iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2252	{
2253	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2254	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2255	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2256	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2257	== RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2258	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2259	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2260	#endif
2261
2262	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2263	uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2264	if (fGstRegs)
2265	{
2266	Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2267	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2268	while (fGstRegs)
2269	{
2270	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2271	fGstRegs &= ~RT_BIT_64(iGstReg);
2272	iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2273	}
2274	}
2275	#else
2276	RT_NOREF(off);
2277	#endif
2278
2279	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2280	pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2281	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2282	}
2283
2284
2285	/**
2286	* Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2287	* and global overview flags.
2288	*/
2289	DECL_FORCE_INLINE(void)
2290	iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2291	{
2292	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2293	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2294	== pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2295	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2296	Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2297	Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2298	Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2299	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2300	Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2301	#endif
2302
2303	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2304	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2305	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2306	#else
2307	RT_NOREF(off);
2308	#endif
2309
2310	uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2311	pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2312	if (!fGstRegShadowsNew)
2313	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2314	pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2315	}
2316
2317
2318	#if 0 /* unused */
2319	/**
2320	* Clear any guest register shadow claim for @a enmGstReg.
2321	*/
2322	DECL_FORCE_INLINE(void)
2323	iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2324	{
2325	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2326	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2327	{
2328	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2329	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2330	}
2331	}
2332	#endif
2333
2334
2335	/**
2336	* Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2337	* as the new shadow of it.
2338	*
2339	* Unlike the other guest reg shadow helpers, this does the logging for you.
2340	* However, it is the liveness state is not asserted here, the caller must do
2341	* that.
2342	*/
2343	DECL_FORCE_INLINE(void)
2344	iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2345	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2346	{
2347	Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2348	if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2349	{
2350	uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2351	Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2352	if (idxHstRegOld == idxHstRegNew)
2353	return;
2354	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2355	g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2356	iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2357	}
2358	else
2359	Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2360	g_aGstShadowInfo[enmGstReg].pszName));
2361	iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2362	}
2363
2364
2365	/**
2366	* Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2367	* to @a idxRegTo.
2368	*/
2369	DECL_FORCE_INLINE(void)
2370	iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2371	IEMNATIVEGSTREG enmGstReg, uint32_t off)
2372	{
2373	Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2374	Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2375	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2376	== pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2377	&& pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2378	Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2379	== pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2380	Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2381	== RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2382
2383	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2384	pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2385	if (!fGstRegShadowsFrom)
2386	pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2387	pReNative->Core.bmHstRegsWithGstShadow \|= RT_BIT_32(idxRegTo);
2388	pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows \|= RT_BIT_64(enmGstReg);
2389	pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2390	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2391	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2392	iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2393	#else
2394	RT_NOREF(off);
2395	#endif
2396	}
2397
2398
2399	/**
2400	* Flushes any delayed guest register writes.
2401	*
2402	* This must be called prior to calling CImpl functions and any helpers that use
2403	* the guest state (like raising exceptions) and such.
2404	*
2405	* This optimization has not yet been implemented. The first target would be
2406	* RIP updates, since these are the most common ones.
2407	*
2408	* @note This function does not flush any shadowing information for guest registers. This needs to be done by
2409	* the caller if it wishes to do so.
2410	*/
2411	DECL_INLINE_THROW(uint32_t)
2412	iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, uint64_t fGstSimdShwExcept = 0)
2413	{
2414	#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2415	uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2416	#else
2417	uint64_t const bmGstRegShadowDirty = 0;
2418	#endif
2419	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2420	uint64_t const bmGstSimdRegShadowDirty = (pReNative->Core.bmGstSimdRegShadowDirtyLo128 \| pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2421	& ~fGstSimdShwExcept;
2422	#else
2423	uint64_t const bmGstSimdRegShadowDirty = 0;
2424	#endif
2425	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2426	uint64_t const fWritebackPc = ~(fGstShwExcept & kIemNativeGstReg_Pc);
2427	#else
2428	uint64_t const fWritebackPc = 0;
2429	#endif
2430	if (bmGstRegShadowDirty \| bmGstSimdRegShadowDirty \| fWritebackPc)
2431	return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2432
2433	return off;
2434	}
2435
2436
2437
2438	/*********************************************************************************************************************************
2439	* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2440	*********************************************************************************************************************************/
2441
2442	#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2443
2444	DECL_FORCE_INLINE(uint8_t)
2445	iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2446	IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2447	{
2448	pReNative->Core.bmHstSimdRegs \|= RT_BIT_32(idxSimdReg);
2449
2450	pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2451	pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2452	pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2453	return idxSimdReg;
2454	}
2455
2456
2457	/**
2458	* Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2459	* SIMD register @a enmGstSimdReg.
2460	*
2461	* ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2462	* host register before calling.
2463	*/
2464	DECL_FORCE_INLINE(void)
2465	iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2466	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2467	{
2468	Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2469	Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2470	Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2471
2472	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2473	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2474	pReNative->Core.bmGstSimdRegShadows \|= RT_BIT_64(enmGstSimdReg);
2475	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxHstSimdReg);
2476	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2477	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2478	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2479	#else
2480	RT_NOREF(off);
2481	#endif
2482	}
2483
2484
2485	/**
2486	* Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2487	* to @a idxSimdRegTo.
2488	*/
2489	DECL_FORCE_INLINE(void)
2490	iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2491	IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2492	{
2493	Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2494	Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2495	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2496	== pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2497	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2498	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2499	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2500	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2501	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2502	Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2503	== pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2504
2505	uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2506	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2507	if (!fGstRegShadowsFrom)
2508	{
2509	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2510	pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2511	}
2512	pReNative->Core.bmHstSimdRegsWithGstShadow \|= RT_BIT_32(idxSimdRegTo);
2513	pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows \|= RT_BIT_64(enmGstSimdReg);
2514	pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2515	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2516	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2517	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2518	#else
2519	RT_NOREF(off);
2520	#endif
2521	}
2522
2523
2524	/**
2525	* Clear any guest register shadow claims from @a idxHstSimdReg.
2526	*
2527	* The register does not need to be shadowing any guest registers.
2528	*/
2529	DECL_FORCE_INLINE(void)
2530	iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2531	{
2532	Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2533	== pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2534	&& pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2535	Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2536	== RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2537	Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2538	&& !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2539
2540	#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2541	uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2542	if (fGstRegs)
2543	{
2544	Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2545	iemNativeDbgInfoAddNativeOffset(pReNative, off);
2546	while (fGstRegs)
2547	{
2548	unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2549	fGstRegs &= ~RT_BIT_64(iGstReg);
2550	iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2551	}
2552	}
2553	#else
2554	RT_NOREF(off);
2555	#endif
2556
2557	pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2558	pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2559	pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2560	pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2561	}
2562
2563	#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2564
2565
2566	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2567	/**
2568	* Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2569	*/
2570	DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2571	{
2572	if (pReNative->Core.offPc)
2573	return iemNativeEmitPcWritebackSlow(pReNative, off);
2574	return off;
2575	}
2576	#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2577
2578
2579	#ifdef IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
2580	/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
2581	* it saves us the trouble of a hidden parameter on MSC/amd64. */
2582	# ifdef RT_ARCH_AMD64
2583	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
2584	# elif defined(RT_ARCH_ARM64)
2585	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
2586	# endif
2587	#endif
2588
2589	#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
2590	extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeFpCtrlRegRestore, (uint64_t u64RegFpCtrl));
2591	#endif
2592
2593	#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
2594
2595	/** @} */
2596
2597	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2598

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 105490

Download in other formats: