VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 105686

Last change on this file since 105686 was 105673, checked in by vboxsync, 3 months ago

VMM/IEM,TM: Do full-TB looping. Redid timer polling in the recompiler. Rewrote the Blt_CheckIrq code, eliminating a conditional. Fixed some TLB related assertions. Moved some IEMCPU members around in hope of better cache-locality. bugref:10656
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 129.1 KB
Line 
1/* $Id: IEMN8veRecompiler.h 105673 2024-08-14 13:57:57Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40#include <iprt/assertcompile.h> /* for RT_IN_ASSEMBLER mode */
41
42/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
43 * Enables generating internal debug info for better TB disassembly dumping. */
44#if defined(DEBUG) || defined(DOXYGEN_RUNNING)
45# define IEMNATIVE_WITH_TB_DEBUG_INFO
46#endif
47
48/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
49 * Enables liveness analysis. */
50#if 1 || defined(DOXYGEN_RUNNING)
51# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
52/*# define IEMLIVENESS_EXTENDED_LAYOUT*/
53#endif
54
55/** @def IEMNATIVE_WITH_EFLAGS_SKIPPING
56 * Enables skipping EFLAGS calculations/updating based on liveness info. */
57#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) || defined(DOXYGEN_RUNNING)
58# define IEMNATIVE_WITH_EFLAGS_SKIPPING
59#endif
60
61/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
62 * Enables strict consistency checks around EFLAGS skipping.
63 * @note Only defined when IEMNATIVE_WITH_EFLAGS_SKIPPING is also defined. */
64#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
65# ifdef VBOX_STRICT
66# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
67# endif
68#elif defined(DOXYGEN_RUNNING)
69# define IEMNATIVE_STRICT_EFLAGS_SKIPPING
70#endif
71
72#ifdef VBOX_WITH_STATISTICS
73/** Always count instructions for now. */
74# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
75#endif
76
77/** @def IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
78 * Enables having only a single prologue for native TBs. */
79#if 1 || defined(DOXYGEN_RUNNING)
80# define IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
81#endif
82
83/** @def IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
84 * Enable this to use common epilogue and tail code for all TBs in a chunk. */
85#if 1 || defined(DOXYGEN_RUNNING)
86# define IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
87#endif
88
89
90/** @name Stack Frame Layout
91 *
92 * @{ */
93/** The size of the area for stack variables and spills and stuff.
94 * @note This limit is duplicated in the python script(s). We add 0x40 for
95 * alignment padding. */
96#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
97/** Number of 64-bit variable slots (0x100 / 8 = 32. */
98#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
99AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
100
101#ifdef RT_ARCH_AMD64
102/** An stack alignment adjustment (between non-volatile register pushes and
103 * the stack variable area, so the latter better aligned). */
104# define IEMNATIVE_FRAME_ALIGN_SIZE 8
105
106/** Number of stack arguments slots for calls made from the frame. */
107# ifdef RT_OS_WINDOWS
108# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
109# else
110# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
111# endif
112/** Number of any shadow arguments (spill area) for calls we make. */
113# ifdef RT_OS_WINDOWS
114# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
115# else
116# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
117# endif
118
119/** Frame pointer (RBP) relative offset of the last push. */
120# ifdef RT_OS_WINDOWS
121# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
122# else
123# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
124# endif
125/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
126 * address for it). */
127# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
128/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
129# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
130/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
131# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
132# ifdef RT_OS_WINDOWS
133/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
134# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
135/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
136# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
137# endif
138
139# ifdef RT_OS_WINDOWS
140/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
141# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
142/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
143# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
144/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
145# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
146/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
147# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
148# endif
149
150#elif RT_ARCH_ARM64
151/** No alignment padding needed for arm64. */
152# define IEMNATIVE_FRAME_ALIGN_SIZE 0
153/** No stack argument slots, got 8 registers for arguments will suffice. */
154# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
155/** There are no argument spill area. */
156# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
157
158/** Number of saved registers at the top of our stack frame.
159 * This includes the return address and old frame pointer, so x19 thru x30. */
160# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
161/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
162# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
163
164/** Frame pointer (BP) relative offset of the last push. */
165# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
166
167/** Frame pointer (BP) relative offset of the stack variable area (the lowest
168 * address for it). */
169# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
170
171#else
172# error "port me"
173#endif
174/** @} */
175
176
177/** @name Fixed Register Allocation(s)
178 * @{ */
179/** @def IEMNATIVE_REG_FIXED_PVMCPU
180 * The number of the register holding the pVCpu pointer. */
181/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
182 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
183 * @note This not available on AMD64, only ARM64. */
184/** @def IEMNATIVE_REG_FIXED_TMP0
185 * Dedicated temporary register.
186 * @todo replace this by a register allocator and content tracker. */
187/** @def IEMNATIVE_REG_FIXED_MASK
188 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
189 * architecture. */
190#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
191/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
192 * Mask SIMD registers with fixes assignments, either by us or dictated by the CPU/OS
193 * architecture. */
194/** @def IEMNATIVE_SIMD_REG_FIXED_TMP0
195 * Dedicated temporary SIMD register. */
196#endif
197#if defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING) /* arm64 goes first because of doxygen */
198# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
199# define IEMNATIVE_REG_FIXED_PVMCPU_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PVMCPU)
200# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
201# define IEMNATIVE_REG_FIXED_PCPUMCTX_ASM RT_CONCAT(x,IEMNATIVE_REG_FIXED_PCPUMCTX)
202# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
203# if defined(IEMNATIVE_WITH_DELAYED_PC_UPDATING) && 0 /* debug the updating with a shadow RIP. */
204# define IEMNATIVE_REG_FIXED_TMP1 ARMV8_A64_REG_X16
205# define IEMNATIVE_REG_FIXED_PC_DBG ARMV8_A64_REG_X26
206# define IEMNATIVE_REG_FIXED_MASK_ADD ( RT_BIT_32(IEMNATIVE_REG_FIXED_TMP1) \
207 | RT_BIT_32(IEMNATIVE_REG_FIXED_PC_DBG))
208# else
209# define IEMNATIVE_REG_FIXED_MASK_ADD 0
210# endif
211# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
212 | RT_BIT_32(ARMV8_A64_REG_LR) \
213 | RT_BIT_32(ARMV8_A64_REG_BP) \
214 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
215 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
216 | RT_BIT_32(ARMV8_A64_REG_X18) \
217 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
218 | IEMNATIVE_REG_FIXED_MASK_ADD)
219
220# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
221# define IEMNATIVE_SIMD_REG_FIXED_TMP0 ARMV8_A64_REG_Q30
222# if defined(IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS)
223# define IEMNATIVE_SIMD_REG_FIXED_MASK RT_BIT_32(ARMV8_A64_REG_Q30)
224# else
225/** @note
226 * ARM64 has 32 registers, but they are only 128-bit wide. So, in order to
227 * support emulating 256-bit registers we pair two real registers statically to
228 * one virtual for now, leaving us with only 16 256-bit registers. We always
229 * pair v0 with v1, v2 with v3, etc. so we mark the higher register as fixed and
230 * the register allocator assumes that it will be always free when the lower is
231 * picked.
232 *
233 * Also ARM64 declares the low 64-bit of v8-v15 as callee saved, so we don't
234 * touch them in order to avoid having to save and restore them in the
235 * prologue/epilogue.
236 */
237# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xff00) \
238 | RT_BIT_32(ARMV8_A64_REG_Q31) \
239 | RT_BIT_32(ARMV8_A64_REG_Q30) \
240 | RT_BIT_32(ARMV8_A64_REG_Q29) \
241 | RT_BIT_32(ARMV8_A64_REG_Q27) \
242 | RT_BIT_32(ARMV8_A64_REG_Q25) \
243 | RT_BIT_32(ARMV8_A64_REG_Q23) \
244 | RT_BIT_32(ARMV8_A64_REG_Q21) \
245 | RT_BIT_32(ARMV8_A64_REG_Q19) \
246 | RT_BIT_32(ARMV8_A64_REG_Q17) \
247 | RT_BIT_32(ARMV8_A64_REG_Q15) \
248 | RT_BIT_32(ARMV8_A64_REG_Q13) \
249 | RT_BIT_32(ARMV8_A64_REG_Q11) \
250 | RT_BIT_32(ARMV8_A64_REG_Q9) \
251 | RT_BIT_32(ARMV8_A64_REG_Q7) \
252 | RT_BIT_32(ARMV8_A64_REG_Q5) \
253 | RT_BIT_32(ARMV8_A64_REG_Q3) \
254 | RT_BIT_32(ARMV8_A64_REG_Q1))
255# endif
256# endif
257
258#elif defined(RT_ARCH_AMD64)
259# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
260# define IEMNATIVE_REG_FIXED_PVMCPU_ASM xBX
261# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
262# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
263 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
264 | RT_BIT_32(X86_GREG_xSP) \
265 | RT_BIT_32(X86_GREG_xBP) )
266
267# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
268# define IEMNATIVE_SIMD_REG_FIXED_TMP0 5 /* xmm5/ymm5 */
269# ifndef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
270# ifndef _MSC_VER
271# define IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
272# endif
273# endif
274# ifdef IEMNATIVE_WITH_SIMD_REG_ACCESS_ALL_REGISTERS
275# define IEMNATIVE_SIMD_REG_FIXED_MASK (RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
276# else
277/** @note On Windows/AMD64 xmm6 through xmm15 are marked as callee saved. */
278# define IEMNATIVE_SIMD_REG_FIXED_MASK ( UINT32_C(0xffc0) \
279 | RT_BIT_32(IEMNATIVE_SIMD_REG_FIXED_TMP0))
280# endif
281# endif
282
283#else
284# error "port me"
285#endif
286/** @} */
287
288/** @name Call related registers.
289 * @{ */
290/** @def IEMNATIVE_CALL_RET_GREG
291 * The return value register. */
292/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
293 * Number of arguments in registers. */
294/** @def IEMNATIVE_CALL_ARG0_GREG
295 * The general purpose register carrying argument \#0. */
296/** @def IEMNATIVE_CALL_ARG1_GREG
297 * The general purpose register carrying argument \#1. */
298/** @def IEMNATIVE_CALL_ARG2_GREG
299 * The general purpose register carrying argument \#2. */
300/** @def IEMNATIVE_CALL_ARG3_GREG
301 * The general purpose register carrying argument \#3. */
302/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
303 * Mask of registers the callee will not save and may trash. */
304#ifdef RT_ARCH_AMD64
305# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
306
307# ifdef RT_OS_WINDOWS
308# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
309# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
310# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
311# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
312# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
313# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
314 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
315 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
316 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
317# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
318 | RT_BIT_32(X86_GREG_xCX) \
319 | RT_BIT_32(X86_GREG_xDX) \
320 | RT_BIT_32(X86_GREG_x8) \
321 | RT_BIT_32(X86_GREG_x9) \
322 | RT_BIT_32(X86_GREG_x10) \
323 | RT_BIT_32(X86_GREG_x11) )
324# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
325/* xmm0 - xmm5 are marked as volatile. */
326# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0x3f))
327# endif
328
329# else /* !RT_OS_WINDOWS */
330# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
331# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
332# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
333# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
334# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
335# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
336# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
337# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
338 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
339 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
340 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
341 | RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
342 | RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
343# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
344 | RT_BIT_32(X86_GREG_xCX) \
345 | RT_BIT_32(X86_GREG_xDX) \
346 | RT_BIT_32(X86_GREG_xDI) \
347 | RT_BIT_32(X86_GREG_xSI) \
348 | RT_BIT_32(X86_GREG_x8) \
349 | RT_BIT_32(X86_GREG_x9) \
350 | RT_BIT_32(X86_GREG_x10) \
351 | RT_BIT_32(X86_GREG_x11) )
352# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
353/* xmm0 - xmm15 are marked as volatile. */
354# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffff))
355# endif
356# endif /* !RT_OS_WINDOWS */
357
358#elif defined(RT_ARCH_ARM64)
359# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
360# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
361# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
362# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
363# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
364# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
365# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
366# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
367# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
368# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
369# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
370 | RT_BIT_32(ARMV8_A64_REG_X1) \
371 | RT_BIT_32(ARMV8_A64_REG_X2) \
372 | RT_BIT_32(ARMV8_A64_REG_X3) \
373 | RT_BIT_32(ARMV8_A64_REG_X4) \
374 | RT_BIT_32(ARMV8_A64_REG_X5) \
375 | RT_BIT_32(ARMV8_A64_REG_X6) \
376 | RT_BIT_32(ARMV8_A64_REG_X7) )
377# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
378 | RT_BIT_32(ARMV8_A64_REG_X1) \
379 | RT_BIT_32(ARMV8_A64_REG_X2) \
380 | RT_BIT_32(ARMV8_A64_REG_X3) \
381 | RT_BIT_32(ARMV8_A64_REG_X4) \
382 | RT_BIT_32(ARMV8_A64_REG_X5) \
383 | RT_BIT_32(ARMV8_A64_REG_X6) \
384 | RT_BIT_32(ARMV8_A64_REG_X7) \
385 | RT_BIT_32(ARMV8_A64_REG_X8) \
386 | RT_BIT_32(ARMV8_A64_REG_X9) \
387 | RT_BIT_32(ARMV8_A64_REG_X10) \
388 | RT_BIT_32(ARMV8_A64_REG_X11) \
389 | RT_BIT_32(ARMV8_A64_REG_X12) \
390 | RT_BIT_32(ARMV8_A64_REG_X13) \
391 | RT_BIT_32(ARMV8_A64_REG_X14) \
392 | RT_BIT_32(ARMV8_A64_REG_X15) \
393 | RT_BIT_32(ARMV8_A64_REG_X16) \
394 | RT_BIT_32(ARMV8_A64_REG_X17) )
395# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
396/* The low 64 bits of v8 - v15 marked as callee saved but the rest is volatile,
397 * so to simplify our life a bit we just mark everything as volatile. */
398# define IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK (UINT32_C(0xffffffff))
399# endif
400
401#endif
402
403/** This is the maximum argument count we'll ever be needing. */
404#define IEMNATIVE_CALL_MAX_ARG_COUNT 7
405#ifdef RT_OS_WINDOWS
406# ifdef VBOXSTRICTRC_STRICT_ENABLED
407# undef IEMNATIVE_CALL_MAX_ARG_COUNT
408# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
409# endif
410#endif
411/** @} */
412
413
414/** @def IEMNATIVE_HST_GREG_COUNT
415 * Number of host general purpose registers we tracker. */
416/** @def IEMNATIVE_HST_GREG_MASK
417 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
418 * inverted register masks and such to get down to a correct set of regs. */
419#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
420/** @def IEMNATIVE_HST_SIMD_REG_COUNT
421 * Number of host SIMD registers we track. */
422/** @def IEMNATIVE_HST_SIMD_REG_MASK
423 * Mask corresponding to IEMNATIVE_HST_SIMD_REG_COUNT that can be applied to
424 * inverted register masks and such to get down to a correct set of regs. */
425#endif
426#ifdef RT_ARCH_AMD64
427# define IEMNATIVE_HST_GREG_COUNT 16
428# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
429
430# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
431# define IEMNATIVE_HST_SIMD_REG_COUNT 16
432# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_C(0xffff)
433# endif
434
435#elif defined(RT_ARCH_ARM64)
436# define IEMNATIVE_HST_GREG_COUNT 32
437# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
438
439# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
440# define IEMNATIVE_HST_SIMD_REG_COUNT 32
441# define IEMNATIVE_HST_SIMD_REG_MASK UINT32_MAX
442# endif
443
444#else
445# error "Port me!"
446#endif
447
448
449#ifndef RT_IN_ASSEMBLER /* ASM-NOINC-START - the rest of the file */
450
451
452/** Native code generator label types. */
453typedef enum
454{
455 kIemNativeLabelType_Invalid = 0,
456 /*
457 * Labels w/o data, only once instance per TB - aka exit reasons.
458 *
459 * Note! Jumps to these requires instructions that are capable of spanning
460 * the max TB length.
461 */
462 /* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
463 kIemNativeLabelType_RaiseDe, /**< Raise (throw) X86_XCPT_DE (00h). */
464 kIemNativeLabelType_RaiseUd, /**< Raise (throw) X86_XCPT_UD (06h). */
465 kIemNativeLabelType_RaiseSseRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. */
466 kIemNativeLabelType_RaiseAvxRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. */
467 kIemNativeLabelType_RaiseSseAvxFpRelated, /**< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. */
468 kIemNativeLabelType_RaiseNm, /**< Raise (throw) X86_XCPT_NM (07h). */
469 kIemNativeLabelType_RaiseGp0, /**< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. */
470 kIemNativeLabelType_RaiseMf, /**< Raise (throw) X86_XCPT_MF (10h). */
471 kIemNativeLabelType_RaiseXf, /**< Raise (throw) X86_XCPT_XF (13h). */
472 kIemNativeLabelType_ObsoleteTb,
473 kIemNativeLabelType_NeedCsLimChecking,
474 kIemNativeLabelType_CheckBranchMiss,
475 kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
476 /* Manually defined labels. */
477 kIemNativeLabelType_ReturnBreak,
478 kIemNativeLabelType_ReturnBreakFF,
479 kIemNativeLabelType_ReturnBreakViaLookup,
480 kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
481 kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
482 kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
483 kIemNativeLabelType_ReturnWithFlags,
484 kIemNativeLabelType_NonZeroRetOrPassUp,
485 kIemNativeLabelType_Return,
486 /** The last fixup for branches that can span almost the whole TB length.
487 * @note Whether kIemNativeLabelType_Return needs to be one of these is
488 * a bit questionable, since nobody jumps to it except other tail code. */
489 kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_Return,
490 /** The last fixup for branches that exits the TB. */
491 kIemNativeLabelType_LastTbExit = kIemNativeLabelType_Return,
492
493 /** Loop-jump target. */
494 kIemNativeLabelType_LoopJumpTarget,
495
496 /*
497 * Labels with data, potentially multiple instances per TB:
498 *
499 * These are localized labels, so no fixed jump type restrictions here.
500 */
501 kIemNativeLabelType_FirstWithMultipleInstances,
502 kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
503 kIemNativeLabelType_Else,
504 kIemNativeLabelType_Endif,
505 kIemNativeLabelType_CheckIrq,
506 kIemNativeLabelType_TlbLookup,
507 kIemNativeLabelType_TlbMiss,
508 kIemNativeLabelType_TlbDone,
509 kIemNativeLabelType_End
510} IEMNATIVELABELTYPE;
511
512#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
513 ((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
514
515
516/** Native code generator label definition. */
517typedef struct IEMNATIVELABEL
518{
519 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
520 * the epilog. */
521 uint32_t off;
522 /** The type of label (IEMNATIVELABELTYPE). */
523 uint16_t enmType;
524 /** Additional label data, type specific. */
525 uint16_t uData;
526} IEMNATIVELABEL;
527/** Pointer to a label. */
528typedef IEMNATIVELABEL *PIEMNATIVELABEL;
529
530
531/** Native code generator fixup types. */
532typedef enum
533{
534 kIemNativeFixupType_Invalid = 0,
535#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
536 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
537 kIemNativeFixupType_Rel32,
538#elif defined(RT_ARCH_ARM64)
539 /** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
540 kIemNativeFixupType_RelImm26At0,
541 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
542 kIemNativeFixupType_RelImm19At5,
543 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
544 kIemNativeFixupType_RelImm14At5,
545#endif
546 kIemNativeFixupType_End
547} IEMNATIVEFIXUPTYPE;
548
549/** Native code generator fixup. */
550typedef struct IEMNATIVEFIXUP
551{
552 /** Code offset of the fixup location. */
553 uint32_t off;
554 /** The IEMNATIVELABEL this is a fixup for. */
555 uint16_t idxLabel;
556 /** The fixup type (IEMNATIVEFIXUPTYPE). */
557 uint8_t enmType;
558 /** Addend or other data. */
559 int8_t offAddend;
560} IEMNATIVEFIXUP;
561/** Pointer to a native code generator fixup. */
562typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
563
564#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
565
566/** Native code generator fixup to per chunk TB tail code. */
567typedef struct IEMNATIVEEXITFIXUP
568{
569 /** Code offset of the fixup location. */
570 uint32_t off;
571 /** The exit reason. */
572 IEMNATIVELABELTYPE enmExitReason;
573} IEMNATIVEEXITFIXUP;
574/** Pointer to a native code generator TB exit fixup. */
575typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
576
577/**
578 * Per executable memory chunk context with addresses for common code.
579 */
580typedef struct IEMNATIVEPERCHUNKCTX
581{
582 /** Pointers to the exit labels */
583 PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
584} IEMNATIVEPERCHUNKCTX;
585/** Pointer to per-chunk recompiler context. */
586typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
587/** Pointer to const per-chunk recompiler context. */
588typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
589
590#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
591
592
593/**
594 * One bit of the state.
595 *
596 * Each register state takes up two bits. We keep the two bits in two separate
597 * 64-bit words to simplify applying them to the guest shadow register mask in
598 * the register allocator.
599 */
600typedef union IEMLIVENESSBIT
601{
602 uint64_t bm64;
603 RT_GCC_EXTENSION struct
604 { /* bit no */
605 uint64_t bmGprs : 16; /**< 0x00 / 0: The 16 general purpose registers. */
606 uint64_t fUnusedPc : 1; /**< 0x10 / 16: (PC in ) */
607 uint64_t fCr0 : 1; /**< 0x11 / 17: */
608 uint64_t fFcw : 1; /**< 0x12 / 18: */
609 uint64_t fFsw : 1; /**< 0x13 / 19: */
610 uint64_t bmSegBase : 6; /**< 0x14 / 20: */
611 uint64_t bmSegAttrib : 6; /**< 0x1a / 26: */
612 uint64_t bmSegLimit : 6; /**< 0x20 / 32: */
613 uint64_t bmSegSel : 6; /**< 0x26 / 38: */
614 uint64_t fCr4 : 1; /**< 0x2c / 44: */
615 uint64_t fXcr0 : 1; /**< 0x2d / 45: */
616 uint64_t fMxCsr : 1; /**< 0x2e / 46: */
617 uint64_t fEflOther : 1; /**< 0x2f / 47: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). First! */
618 uint64_t fEflCf : 1; /**< 0x30 / 48: Carry flag (X86_EFL_CF / 0). */
619 uint64_t fEflPf : 1; /**< 0x31 / 49: Parity flag (X86_EFL_PF / 2). */
620 uint64_t fEflAf : 1; /**< 0x32 / 50: Auxilary carry flag (X86_EFL_AF / 4). */
621 uint64_t fEflZf : 1; /**< 0x33 / 51: Zero flag (X86_EFL_ZF / 6). */
622 uint64_t fEflSf : 1; /**< 0x34 / 52: Signed flag (X86_EFL_SF / 7). */
623 uint64_t fEflOf : 1; /**< 0x35 / 53: Overflow flag (X86_EFL_OF / 12). */
624 uint64_t uUnused : 10; /* 0x36 / 54 -> 0x40/64 */
625 };
626} IEMLIVENESSBIT;
627AssertCompileSize(IEMLIVENESSBIT, 8);
628
629#define IEMLIVENESSBIT_IDX_EFL_OTHER ((unsigned)kIemNativeGstReg_EFlags + 0)
630#define IEMLIVENESSBIT_IDX_EFL_CF ((unsigned)kIemNativeGstReg_EFlags + 1)
631#define IEMLIVENESSBIT_IDX_EFL_PF ((unsigned)kIemNativeGstReg_EFlags + 2)
632#define IEMLIVENESSBIT_IDX_EFL_AF ((unsigned)kIemNativeGstReg_EFlags + 3)
633#define IEMLIVENESSBIT_IDX_EFL_ZF ((unsigned)kIemNativeGstReg_EFlags + 4)
634#define IEMLIVENESSBIT_IDX_EFL_SF ((unsigned)kIemNativeGstReg_EFlags + 5)
635#define IEMLIVENESSBIT_IDX_EFL_OF ((unsigned)kIemNativeGstReg_EFlags + 6)
636
637
638/**
639 * A liveness state entry.
640 *
641 * The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
642 * Once we add a SSE register shadowing, we'll add another 64-bit element for
643 * that.
644 */
645typedef union IEMLIVENESSENTRY
646{
647#ifndef IEMLIVENESS_EXTENDED_LAYOUT
648 uint64_t bm64[16 / 8];
649 uint16_t bm32[16 / 4];
650 uint16_t bm16[16 / 2];
651 uint8_t bm8[ 16 / 1];
652 IEMLIVENESSBIT aBits[2];
653#else
654 uint64_t bm64[32 / 8];
655 uint16_t bm32[32 / 4];
656 uint16_t bm16[32 / 2];
657 uint8_t bm8[ 32 / 1];
658 IEMLIVENESSBIT aBits[4];
659#endif
660 RT_GCC_EXTENSION struct
661 {
662 /** Bit \#0 of the register states. */
663 IEMLIVENESSBIT Bit0;
664 /** Bit \#1 of the register states. */
665 IEMLIVENESSBIT Bit1;
666#ifdef IEMLIVENESS_EXTENDED_LAYOUT
667 /** Bit \#2 of the register states. */
668 IEMLIVENESSBIT Bit2;
669 /** Bit \#3 of the register states. */
670 IEMLIVENESSBIT Bit3;
671#endif
672 };
673} IEMLIVENESSENTRY;
674#ifndef IEMLIVENESS_EXTENDED_LAYOUT
675AssertCompileSize(IEMLIVENESSENTRY, 16);
676#else
677AssertCompileSize(IEMLIVENESSENTRY, 32);
678#endif
679/** Pointer to a liveness state entry. */
680typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
681/** Pointer to a const liveness state entry. */
682typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
683
684/** @name 64-bit value masks for IEMLIVENESSENTRY.
685 * @{ */ /* 0xzzzzyyyyxxxxwwww */
686#define IEMLIVENESSBIT_MASK UINT64_C(0x003ffffffffeffff)
687
688#ifndef IEMLIVENESS_EXTENDED_LAYOUT
689# define IEMLIVENESSBIT0_XCPT_OR_CALL UINT64_C(0x0000000000000000)
690# define IEMLIVENESSBIT1_XCPT_OR_CALL IEMLIVENESSBIT_MASK
691
692# define IEMLIVENESSBIT0_ALL_UNUSED IEMLIVENESSBIT_MASK
693# define IEMLIVENESSBIT1_ALL_UNUSED UINT64_C(0x0000000000000000)
694#endif
695
696#define IEMLIVENESSBIT_ALL_EFL_MASK UINT64_C(0x003f800000000000)
697#define IEMLIVENESSBIT_STATUS_EFL_MASK UINT64_C(0x003f000000000000)
698
699#ifndef IEMLIVENESS_EXTENDED_LAYOUT
700# define IEMLIVENESSBIT0_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
701# define IEMLIVENESSBIT1_ALL_EFL_INPUT IEMLIVENESSBIT_ALL_EFL_MASK
702#endif
703/** @} */
704
705
706/** @name The liveness state for a register.
707 *
708 * The state values have been picked to with state accumulation in mind (what
709 * the iemNativeLivenessFunc_xxxx functions does), as that is the most
710 * performance critical work done with the values.
711 *
712 * This is a compressed state that only requires 2 bits per register.
713 * When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
714 * 1. the incoming state from the following call,
715 * 2. the outgoing state for this call,
716 * 3. mask of the entries set in the 2nd.
717 *
718 * The mask entry (3rd one above) will be used both when updating the outgoing
719 * state and when merging in incoming state for registers not touched by the
720 * current call.
721 *
722 * @{ */
723#ifndef IEMLIVENESS_EXTENDED_LAYOUT
724/** The register will be clobbered and the current value thrown away.
725 *
726 * When this is applied to the state (2) we'll simply be AND'ing it with the
727 * (old) mask (3) and adding the register to the mask. This way we'll
728 * preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
729 * IEMLIVENESS_STATE_INPUT states. */
730# define IEMLIVENESS_STATE_CLOBBERED 0
731/** The register is unused in the remainder of the TB.
732 *
733 * This is an initial state and can not be set by any of the
734 * iemNativeLivenessFunc_xxxx callbacks. */
735# define IEMLIVENESS_STATE_UNUSED 1
736/** The register value is required in a potential call or exception.
737 *
738 * This means that the register value must be calculated and is best written to
739 * the state, but that any shadowing registers can be flushed thereafter as it's
740 * not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
741 *
742 * It is typically applied across the board, but we preserve incoming
743 * IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
744 * trickery to filter out IEMLIVENESS_STATE_UNUSED:
745 * 1. r0 = old & ~mask;
746 * 2. r0 = t1 & (t1 >> 1)'
747 * 3. state |= r0 | 0b10;
748 * 4. mask = ~0;
749 */
750# define IEMLIVENESS_STATE_XCPT_OR_CALL 2
751/** The register value is used as input.
752 *
753 * This means that the register value must be calculated and it is best to keep
754 * it in a register. It does not need to be writtent out as such. This is the
755 * highest priority state.
756 *
757 * Whether the call modifies the register or not isn't relevant to earlier
758 * calls, so that's not recorded.
759 *
760 * When applying this state we just or in the value in the outgoing state and
761 * mask. */
762# define IEMLIVENESS_STATE_INPUT 3
763/** Mask of the state bits. */
764# define IEMLIVENESS_STATE_MASK 3
765/** The number of bits per state. */
766# define IEMLIVENESS_STATE_BIT_COUNT 2
767/** Check if we're expecting read & write accesses to a register with the given (previous) liveness state. */
768# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) ((uint32_t)((a_uState) - 1U) >= (uint32_t)(IEMLIVENESS_STATE_INPUT - 1U))
769/** Check if we're expecting read accesses to a register with the given (previous) liveness state. */
770# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState)
771/** Check if a register clobbering is expected given the (previous) liveness state.
772 * The state must be either CLOBBERED or XCPT_OR_CALL, but it may also
773 * include INPUT if the register is used in more than one place. */
774# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) ((uint32_t)(a_uState) != IEMLIVENESS_STATE_UNUSED)
775
776/** Check if all status flags are going to be clobbered and doesn't need
777 * calculating in the current step.
778 * @param a_pCurEntry The current liveness entry. */
779# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
780 ( (((a_pCurEntry)->Bit0.bm64 | (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
781
782#else /* IEMLIVENESS_EXTENDED_LAYOUT */
783/** The register is not used any more. */
784# define IEMLIVENESS_STATE_UNUSED 0
785/** Flag: The register is required in a potential exception or call. */
786# define IEMLIVENESS_STATE_POT_XCPT_OR_CALL 1
787# define IEMLIVENESS_BIT_POT_XCPT_OR_CALL 0
788/** Flag: The register is read. */
789# define IEMLIVENESS_STATE_READ 2
790# define IEMLIVENESS_BIT_READ 1
791/** Flag: The register is written. */
792# define IEMLIVENESS_STATE_WRITE 4
793# define IEMLIVENESS_BIT_WRITE 2
794/** Flag: Unconditional call (not needed, can be redefined for research). */
795# define IEMLIVENESS_STATE_CALL 8
796# define IEMLIVENESS_BIT_CALL 3
797# define IEMLIVENESS_BIT_OTHER 3 /**< More convenient name for this one. */
798# define IEMLIVENESS_STATE_IS_MODIFY_EXPECTED(a_uState) \
799 ( ((a_uState) & (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ)) == (IEMLIVENESS_STATE_WRITE | IEMLIVENESS_STATE_READ) )
800# define IEMLIVENESS_STATE_IS_INPUT_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_READ)
801# define IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(a_uState) RT_BOOL((a_uState) & IEMLIVENESS_STATE_WRITE)
802
803# define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
804 ( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 & IEMLIVENESSBIT_STATUS_EFL_MASK) == IEMLIVENESSBIT_STATUS_EFL_MASK \
805 && !( ((a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64) \
806 & IEMLIVENESSBIT_STATUS_EFL_MASK) )
807
808#endif /* IEMLIVENESS_EXTENDED_LAYOUT */
809/** @} */
810
811/** @name Liveness helpers for builtin functions and similar.
812 *
813 * These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
814 * own set of manimulator macros for those.
815 *
816 * @{ */
817/** Initializing the state as all unused. */
818#ifndef IEMLIVENESS_EXTENDED_LAYOUT
819# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
820 do { \
821 (a_pOutgoing)->Bit0.bm64 = IEMLIVENESSBIT0_ALL_UNUSED; \
822 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_ALL_UNUSED; \
823 } while (0)
824#else
825# define IEM_LIVENESS_RAW_INIT_AS_UNUSED(a_pOutgoing) \
826 do { \
827 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = 0; \
828 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = 0; \
829 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
830 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
831 } while (0)
832#endif
833
834/** Initializing the outgoing state with a potential xcpt or call state.
835 * This only works when all later changes will be IEMLIVENESS_STATE_INPUT. */
836#ifndef IEMLIVENESS_EXTENDED_LAYOUT
837# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
838 do { \
839 (a_pOutgoing)->Bit0.bm64 = (a_pIncoming)->Bit0.bm64 & (a_pIncoming)->Bit1.bm64; \
840 (a_pOutgoing)->Bit1.bm64 = IEMLIVENESSBIT1_XCPT_OR_CALL; \
841 } while (0)
842#else
843# define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
844 do { \
845 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_POT_XCPT_OR_CALL].bm64 = IEMLIVENESSBIT_MASK; \
846 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ ].bm64 = (a_pIncoming)->aBits[IEMLIVENESS_BIT_READ].bm64; \
847 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_WRITE ].bm64 = 0; \
848 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_OTHER ].bm64 = 0; \
849 } while (0)
850#endif
851
852/** Adds a segment base register as input to the outgoing state. */
853#ifndef IEMLIVENESS_EXTENDED_LAYOUT
854# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
855 (a_pOutgoing)->Bit0.bmSegBase |= RT_BIT_64(a_iSReg); \
856 (a_pOutgoing)->Bit1.bmSegBase |= RT_BIT_64(a_iSReg); \
857 } while (0)
858#else
859# define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) do { \
860 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegBase |= RT_BIT_64(a_iSReg); \
861 } while (0)
862#endif
863
864/** Adds a segment attribute register as input to the outgoing state. */
865#ifndef IEMLIVENESS_EXTENDED_LAYOUT
866# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
867 (a_pOutgoing)->Bit0.bmSegAttrib |= RT_BIT_64(a_iSReg); \
868 (a_pOutgoing)->Bit1.bmSegAttrib |= RT_BIT_64(a_iSReg); \
869 } while (0)
870#else
871# define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) do { \
872 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegAttrib |= RT_BIT_64(a_iSReg); \
873 } while (0)
874#endif
875
876/** Adds a segment limit register as input to the outgoing state. */
877#ifndef IEMLIVENESS_EXTENDED_LAYOUT
878# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
879 (a_pOutgoing)->Bit0.bmSegLimit |= RT_BIT_64(a_iSReg); \
880 (a_pOutgoing)->Bit1.bmSegLimit |= RT_BIT_64(a_iSReg); \
881 } while (0)
882#else
883# define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) do { \
884 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].bmSegLimit |= RT_BIT_64(a_iSReg); \
885 } while (0)
886#endif
887
888/** Adds a segment limit register as input to the outgoing state. */
889#ifndef IEMLIVENESS_EXTENDED_LAYOUT
890# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
891 (a_pOutgoing)->Bit0.a_fEflMember |= 1; \
892 (a_pOutgoing)->Bit1.a_fEflMember |= 1; \
893 } while (0)
894#else
895# define IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(a_pOutgoing, a_fEflMember) do { \
896 (a_pOutgoing)->aBits[IEMLIVENESS_BIT_READ].a_fEflMember |= 1; \
897 } while (0)
898#endif
899/** @} */
900
901/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
902 * Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
903 * calculated and up to date. This is to double check that we haven't skipped
904 * EFLAGS calculations when we actually need them. NOP in non-strict builds.
905 * @note has to be placed in
906 */
907#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
908# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
909 do { (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); } while (0)
910#else
911# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
912#endif
913
914
915/**
916 * Guest registers that can be shadowed in GPRs.
917 *
918 * This runs parallel to the liveness state (IEMLIVENESSBIT, ++). The EFlags
919 * must be placed last, as the liveness state tracks it as 7 subcomponents and
920 * we don't want to waste space here.
921 *
922 * @note Make sure to update IEMLIVENESSBIT, IEMLIVENESSBIT_ALL_EFL_MASK and
923 * friends as well as IEMAllN8veLiveness.cpp.
924 */
925typedef enum IEMNATIVEGSTREG : uint8_t
926{
927 kIemNativeGstReg_GprFirst = 0,
928 kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
929 kIemNativeGstReg_Pc,
930 kIemNativeGstReg_Cr0,
931 kIemNativeGstReg_FpuFcw,
932 kIemNativeGstReg_FpuFsw,
933 kIemNativeGstReg_SegBaseFirst,
934 kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
935 kIemNativeGstReg_SegAttribFirst,
936 kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
937 kIemNativeGstReg_SegLimitFirst,
938 kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
939 kIemNativeGstReg_SegSelFirst,
940 kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
941 kIemNativeGstReg_Cr4,
942 kIemNativeGstReg_Xcr0,
943 kIemNativeGstReg_MxCsr,
944 kIemNativeGstReg_EFlags, /**< 32-bit, includes internal flags - last! */
945 kIemNativeGstReg_End
946} IEMNATIVEGSTREG;
947AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
948AssertCompile((UINT64_C(0x7f) << kIemNativeGstReg_EFlags) == IEMLIVENESSBIT_ALL_EFL_MASK);
949
950/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
951 * @{ */
952#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
953#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
954#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
955#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
956#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
957/** @} */
958
959#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
960
961/**
962 * Guest registers that can be shadowed in host SIMD registers.
963 *
964 * @todo r=aeichner Liveness tracking
965 * @todo r=aeichner Given that we can only track xmm/ymm here does this actually make sense?
966 */
967typedef enum IEMNATIVEGSTSIMDREG : uint8_t
968{
969 kIemNativeGstSimdReg_SimdRegFirst = 0,
970 kIemNativeGstSimdReg_SimdRegLast = kIemNativeGstSimdReg_SimdRegFirst + 15,
971 kIemNativeGstSimdReg_End
972} IEMNATIVEGSTSIMDREG;
973
974/** @name Helpers for converting register numbers to IEMNATIVEGSTSIMDREG values.
975 * @{ */
976#define IEMNATIVEGSTSIMDREG_SIMD(a_iSimdReg) ((IEMNATIVEGSTSIMDREG)(kIemNativeGstSimdReg_SimdRegFirst + (a_iSimdReg)))
977/** @} */
978
979/**
980 * The Load/store size for a SIMD guest register.
981 */
982typedef enum IEMNATIVEGSTSIMDREGLDSTSZ : uint8_t
983{
984 /** Invalid size. */
985 kIemNativeGstSimdRegLdStSz_Invalid = 0,
986 /** Loads the low 128-bit of a guest SIMD register. */
987 kIemNativeGstSimdRegLdStSz_Low128,
988 /** Loads the high 128-bit of a guest SIMD register. */
989 kIemNativeGstSimdRegLdStSz_High128,
990 /** Loads the whole 256-bits of a guest SIMD register. */
991 kIemNativeGstSimdRegLdStSz_256,
992 /** End value. */
993 kIemNativeGstSimdRegLdStSz_End
994} IEMNATIVEGSTSIMDREGLDSTSZ;
995
996#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
997
998/**
999 * Intended use statement for iemNativeRegAllocTmpForGuestReg().
1000 */
1001typedef enum IEMNATIVEGSTREGUSE
1002{
1003 /** The usage is read-only, the register holding the guest register
1004 * shadow copy will not be modified by the caller. */
1005 kIemNativeGstRegUse_ReadOnly = 0,
1006 /** The caller will update the guest register (think: PC += cbInstr).
1007 * The guest shadow copy will follow the returned register. */
1008 kIemNativeGstRegUse_ForUpdate,
1009 /** The call will put an entirely new value in the guest register, so
1010 * if new register is allocate it will be returned uninitialized. */
1011 kIemNativeGstRegUse_ForFullWrite,
1012 /** The caller will use the guest register value as input in a calculation
1013 * and the host register will be modified.
1014 * This means that the returned host register will not be marked as a shadow
1015 * copy of the guest register. */
1016 kIemNativeGstRegUse_Calculation
1017} IEMNATIVEGSTREGUSE;
1018
1019/**
1020 * Guest registers (classes) that can be referenced.
1021 */
1022typedef enum IEMNATIVEGSTREGREF : uint8_t
1023{
1024 kIemNativeGstRegRef_Invalid = 0,
1025 kIemNativeGstRegRef_Gpr,
1026 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
1027 kIemNativeGstRegRef_EFlags,
1028 kIemNativeGstRegRef_MxCsr,
1029 kIemNativeGstRegRef_FpuReg,
1030 kIemNativeGstRegRef_MReg,
1031 kIemNativeGstRegRef_XReg,
1032 kIemNativeGstRegRef_X87,
1033 kIemNativeGstRegRef_XState,
1034 //kIemNativeGstRegRef_YReg, - doesn't work.
1035 kIemNativeGstRegRef_End
1036} IEMNATIVEGSTREGREF;
1037
1038
1039/** Variable kinds. */
1040typedef enum IEMNATIVEVARKIND : uint8_t
1041{
1042 /** Customary invalid zero value. */
1043 kIemNativeVarKind_Invalid = 0,
1044 /** This is either in a register or on the stack. */
1045 kIemNativeVarKind_Stack,
1046 /** Immediate value - loaded into register when needed, or can live on the
1047 * stack if referenced (in theory). */
1048 kIemNativeVarKind_Immediate,
1049 /** Variable reference - loaded into register when needed, never stack. */
1050 kIemNativeVarKind_VarRef,
1051 /** Guest register reference - loaded into register when needed, never stack. */
1052 kIemNativeVarKind_GstRegRef,
1053 /** End of valid values. */
1054 kIemNativeVarKind_End
1055} IEMNATIVEVARKIND;
1056
1057
1058/** Variable or argument. */
1059typedef struct IEMNATIVEVAR
1060{
1061 /** The kind of variable. */
1062 IEMNATIVEVARKIND enmKind;
1063 /** The variable size in bytes. */
1064 uint8_t cbVar;
1065 /** The first stack slot (uint64_t), except for immediate and references
1066 * where it usually is UINT8_MAX. This is allocated lazily, so if a variable
1067 * has a stack slot it has been initialized and has a value. Unused variables
1068 * has neither a stack slot nor a host register assignment. */
1069 uint8_t idxStackSlot;
1070 /** The host register allocated for the variable, UINT8_MAX if not. */
1071 uint8_t idxReg;
1072 /** The argument number if argument, UINT8_MAX if regular variable. */
1073 uint8_t uArgNo;
1074 /** If referenced, the index (unpacked) of the variable referencing this one,
1075 * otherwise UINT8_MAX. A referenced variable must only be placed on the stack
1076 * and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
1077 uint8_t idxReferrerVar;
1078 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
1079 * @todo not sure what this really is for... */
1080 IEMNATIVEGSTREG enmGstReg;
1081#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1082 /** Flag whether this variable is held in a SIMD register (only supported for 128-bit and 256-bit variables),
1083 * only valid when idxReg is not UINT8_MAX. */
1084 bool fSimdReg : 1;
1085 /** Set if the registered is currently used exclusively, false if the
1086 * variable is idle and the register can be grabbed. */
1087 bool fRegAcquired : 1;
1088#else
1089 /** Set if the registered is currently used exclusively, false if the
1090 * variable is idle and the register can be grabbed. */
1091 bool fRegAcquired;
1092#endif
1093
1094 union
1095 {
1096 /** kIemNativeVarKind_Immediate: The immediate value. */
1097 uint64_t uValue;
1098 /** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
1099 uint8_t idxRefVar;
1100 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
1101 struct
1102 {
1103 /** The class of register. */
1104 IEMNATIVEGSTREGREF enmClass;
1105 /** Index within the class. */
1106 uint8_t idx;
1107 } GstRegRef;
1108 } u;
1109} IEMNATIVEVAR;
1110/** Pointer to a variable or argument. */
1111typedef IEMNATIVEVAR *PIEMNATIVEVAR;
1112/** Pointer to a const variable or argument. */
1113typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
1114
1115/** What is being kept in a host register. */
1116typedef enum IEMNATIVEWHAT : uint8_t
1117{
1118 /** The traditional invalid zero value. */
1119 kIemNativeWhat_Invalid = 0,
1120 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
1121 kIemNativeWhat_Var,
1122 /** Temporary register, this is typically freed when a MC completes. */
1123 kIemNativeWhat_Tmp,
1124 /** Call argument w/o a variable mapping. This is free (via
1125 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
1126 kIemNativeWhat_Arg,
1127 /** Return status code.
1128 * @todo not sure if we need this... */
1129 kIemNativeWhat_rc,
1130 /** The fixed pVCpu (PVMCPUCC) register.
1131 * @todo consider offsetting this on amd64 to use negative offsets to access
1132 * more members using 8-byte disp. */
1133 kIemNativeWhat_pVCpuFixed,
1134 /** The fixed pCtx (PCPUMCTX) register.
1135 * @todo consider offsetting this on amd64 to use negative offsets to access
1136 * more members using 8-byte disp. */
1137 kIemNativeWhat_pCtxFixed,
1138 /** Fixed temporary register. */
1139 kIemNativeWhat_FixedTmp,
1140#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1141 /** Shadow RIP for the delayed RIP updating debugging. */
1142 kIemNativeWhat_PcShadow,
1143#endif
1144 /** Register reserved by the CPU or OS architecture. */
1145 kIemNativeWhat_FixedReserved,
1146 /** End of valid values. */
1147 kIemNativeWhat_End
1148} IEMNATIVEWHAT;
1149
1150/**
1151 * Host general register entry.
1152 *
1153 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
1154 *
1155 * @todo Track immediate values in host registers similarlly to how we track the
1156 * guest register shadow copies. For it to be real helpful, though,
1157 * we probably need to know which will be reused and put them into
1158 * non-volatile registers, otherwise it's going to be more or less
1159 * restricted to an instruction or two.
1160 */
1161typedef struct IEMNATIVEHSTREG
1162{
1163 /** Set of guest registers this one shadows.
1164 *
1165 * Using a bitmap here so we can designate the same host register as a copy
1166 * for more than one guest register. This is expected to be useful in
1167 * situations where one value is copied to several registers in a sequence.
1168 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1169 * sequence we'd want to let this register follow to be a copy of and there
1170 * will always be places where we'd be picking the wrong one.
1171 */
1172 uint64_t fGstRegShadows;
1173 /** What is being kept in this register. */
1174 IEMNATIVEWHAT enmWhat;
1175 /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1176 uint8_t idxVar;
1177 /** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
1178 * by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
1179 * that scope. */
1180 uint8_t idxStackSlot;
1181 /** Alignment padding. */
1182 uint8_t abAlign[5];
1183} IEMNATIVEHSTREG;
1184
1185
1186#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1187/**
1188 * Host SIMD register entry - this tracks a virtual 256-bit register split into two 128-bit
1189 * halves, on architectures where there is no 256-bit register available this entry will track
1190 * two adjacent 128-bit host registers.
1191 *
1192 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstSimdRegs.
1193 */
1194typedef struct IEMNATIVEHSTSIMDREG
1195{
1196 /** Set of guest registers this one shadows.
1197 *
1198 * Using a bitmap here so we can designate the same host register as a copy
1199 * for more than one guest register. This is expected to be useful in
1200 * situations where one value is copied to several registers in a sequence.
1201 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
1202 * sequence we'd want to let this register follow to be a copy of and there
1203 * will always be places where we'd be picking the wrong one.
1204 */
1205 uint64_t fGstRegShadows;
1206 /** What is being kept in this register. */
1207 IEMNATIVEWHAT enmWhat;
1208 /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
1209 uint8_t idxVar;
1210 /** Flag what is currently loaded, low 128-bits, high 128-bits or complete 256-bits. */
1211 IEMNATIVEGSTSIMDREGLDSTSZ enmLoaded;
1212 /** Alignment padding. */
1213 uint8_t abAlign[5];
1214} IEMNATIVEHSTSIMDREG;
1215#endif
1216
1217
1218/**
1219 * Core state for the native recompiler, that is, things that needs careful
1220 * handling when dealing with branches.
1221 */
1222typedef struct IEMNATIVECORESTATE
1223{
1224#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1225 /** The current instruction offset in bytes from when the guest program counter
1226 * was updated last. Used for delaying the write to the guest context program counter
1227 * as long as possible. */
1228 uint32_t offPc;
1229 /** Number of instructions where we could skip the updating. */
1230 uint32_t cInstrPcUpdateSkipped;
1231#endif
1232 /** Allocation bitmap for aHstRegs. */
1233 uint32_t bmHstRegs;
1234
1235 /** Bitmap marking which host register contains guest register shadow copies.
1236 * This is used during register allocation to try preserve copies. */
1237 uint32_t bmHstRegsWithGstShadow;
1238 /** Bitmap marking valid entries in aidxGstRegShadows. */
1239 uint64_t bmGstRegShadows;
1240#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1241 /** Bitmap marking the shadowed guest register as dirty and needing writeback when flushing. */
1242 uint64_t bmGstRegShadowDirty;
1243#endif
1244
1245#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1246 /** Allocation bitmap for aHstSimdRegs. */
1247 uint32_t bmHstSimdRegs;
1248
1249 /** Bitmap marking which host SIMD register contains guest SIMD register shadow copies.
1250 * This is used during register allocation to try preserve copies. */
1251 uint32_t bmHstSimdRegsWithGstShadow;
1252 /** Bitmap marking valid entries in aidxSimdGstRegShadows. */
1253 uint64_t bmGstSimdRegShadows;
1254 /** Bitmap marking whether the low 128-bit of the shadowed guest register are dirty and need writeback. */
1255 uint64_t bmGstSimdRegShadowDirtyLo128;
1256 /** Bitmap marking whether the high 128-bit of the shadowed guest register are dirty and need writeback. */
1257 uint64_t bmGstSimdRegShadowDirtyHi128;
1258#endif
1259
1260 union
1261 {
1262 /** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
1263 uint8_t aidxArgVars[8];
1264 /** For more efficient resetting. */
1265 uint64_t u64ArgVars;
1266 };
1267
1268 /** Allocation bitmap for the stack. */
1269 uint32_t bmStack;
1270 /** Allocation bitmap for aVars. */
1271 uint32_t bmVars;
1272
1273 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
1274 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
1275 * (A shadow copy of a guest register can only be held in a one host register,
1276 * there are no duplicate copies or ambiguities like that). */
1277 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
1278#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1279 /** Maps a guest SIMD register to a host SIMD register (index by IEMNATIVEGSTSIMDREG).
1280 * Entries are only valid if the corresponding bit in bmGstSimdRegShadows is set.
1281 * (A shadow copy of a guest register can only be held in a one host register,
1282 * there are no duplicate copies or ambiguities like that). */
1283 uint8_t aidxGstSimdRegShadows[kIemNativeGstSimdReg_End];
1284#endif
1285
1286 /** Host register allocation tracking. */
1287 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
1288#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1289 /** Host SIMD register allocation tracking. */
1290 IEMNATIVEHSTSIMDREG aHstSimdRegs[IEMNATIVE_HST_SIMD_REG_COUNT];
1291#endif
1292
1293 /** Variables and arguments. */
1294 IEMNATIVEVAR aVars[9];
1295} IEMNATIVECORESTATE;
1296/** Pointer to core state. */
1297typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
1298/** Pointer to const core state. */
1299typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
1300
1301/** @def IEMNATIVE_VAR_IDX_UNPACK
1302 * @returns Index into IEMNATIVECORESTATE::aVars.
1303 * @param a_idxVar Variable index w/ magic (in strict builds).
1304 */
1305/** @def IEMNATIVE_VAR_IDX_PACK
1306 * @returns Variable index w/ magic (in strict builds).
1307 * @param a_idxVar Index into IEMNATIVECORESTATE::aVars.
1308 */
1309#ifdef VBOX_STRICT
1310# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
1311# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) ((a_idxVar) | IEMNATIVE_VAR_IDX_MAGIC)
1312# define IEMNATIVE_VAR_IDX_MAGIC UINT8_C(0xd0)
1313# define IEMNATIVE_VAR_IDX_MAGIC_MASK UINT8_C(0xf0)
1314# define IEMNATIVE_VAR_IDX_MASK UINT8_C(0x0f)
1315#else
1316# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
1317# define IEMNATIVE_VAR_IDX_PACK(a_idxVar) (a_idxVar)
1318#endif
1319
1320
1321#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1322/** Clear the dirty state of the given guest SIMD register. */
1323# define IEMNATIVE_SIMD_REG_STATE_CLR_DIRTY(a_pReNative, a_iSimdReg) \
1324 do { \
1325 (a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 &= ~RT_BIT_64(a_iSimdReg); \
1326 (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 &= ~RT_BIT_64(a_iSimdReg); \
1327 } while (0)
1328
1329/** Returns whether the low 128-bits of the given guest SIMD register are dirty. */
1330# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1331 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 & RT_BIT_64(a_iSimdReg))
1332/** Returns whether the high 128-bits of the given guest SIMD register are dirty. */
1333# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1334 RT_BOOL((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 & RT_BIT_64(a_iSimdReg))
1335/** Returns whether the given guest SIMD register is dirty. */
1336# define IEMNATIVE_SIMD_REG_STATE_IS_DIRTY_U256(a_pReNative, a_iSimdReg) \
1337 RT_BOOL(((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 | (a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128) & RT_BIT_64(a_iSimdReg))
1338
1339/** Set the low 128-bits of the given guest SIMD register to the dirty state. */
1340# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_LO_U128(a_pReNative, a_iSimdReg) \
1341 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyLo128 |= RT_BIT_64(a_iSimdReg))
1342/** Set the high 128-bits of the given guest SIMD register to the dirty state. */
1343# define IEMNATIVE_SIMD_REG_STATE_SET_DIRTY_HI_U128(a_pReNative, a_iSimdReg) \
1344 ((a_pReNative)->Core.bmGstSimdRegShadowDirtyHi128 |= RT_BIT_64(a_iSimdReg))
1345
1346/** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1347# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE RT_BIT_32(0)
1348 /** Flag for indicating that IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() has emitted code in the current TB. */
1349# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE RT_BIT_32(1)
1350/** Flag for indicating that IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() has emitted code in the current TB. */
1351# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE RT_BIT_32(2)
1352/** Flag for indicating that IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() has emitted code in the current TB. */
1353# define IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX RT_BIT_32(3)
1354# ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
1355/** Flag indicating that the guest MXCSR was synced to the host floating point control register. */
1356# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED RT_BIT_32(4)
1357/** Flag indicating whether the host floating point control register was saved before overwriting it. */
1358# define IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SAVED RT_BIT_32(5)
1359# endif
1360#endif
1361
1362
1363/**
1364 * Conditional stack entry.
1365 */
1366typedef struct IEMNATIVECOND
1367{
1368 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
1369 bool fInElse;
1370 /** The label for the IEM_MC_ELSE. */
1371 uint32_t idxLabelElse;
1372 /** The label for the IEM_MC_ENDIF. */
1373 uint32_t idxLabelEndIf;
1374 /** The initial state snapshot as the if-block starts executing. */
1375 IEMNATIVECORESTATE InitialState;
1376 /** The state snapshot at the end of the if-block. */
1377 IEMNATIVECORESTATE IfFinalState;
1378} IEMNATIVECOND;
1379/** Pointer to a condition stack entry. */
1380typedef IEMNATIVECOND *PIEMNATIVECOND;
1381
1382
1383/**
1384 * Native recompiler state.
1385 */
1386typedef struct IEMRECOMPILERSTATE
1387{
1388 /** Size of the buffer that pbNativeRecompileBufR3 points to in
1389 * IEMNATIVEINSTR units. */
1390 uint32_t cInstrBufAlloc;
1391#ifdef VBOX_STRICT
1392 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
1393 uint32_t offInstrBufChecked;
1394#else
1395 uint32_t uPadding1; /* We don't keep track of the size here... */
1396#endif
1397 /** Fixed temporary code buffer for native recompilation. */
1398 PIEMNATIVEINSTR pInstrBuf;
1399
1400 /** Bitmaps with the label types used. */
1401 uint64_t bmLabelTypes;
1402 /** Actual number of labels in paLabels. */
1403 uint32_t cLabels;
1404 /** Max number of entries allowed in paLabels before reallocating it. */
1405 uint32_t cLabelsAlloc;
1406 /** Labels defined while recompiling (referenced by fixups). */
1407 PIEMNATIVELABEL paLabels;
1408 /** Array with indexes of unique labels (uData always 0). */
1409 uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
1410
1411 /** Actual number of fixups paFixups. */
1412 uint32_t cFixups;
1413 /** Max number of entries allowed in paFixups before reallocating it. */
1414 uint32_t cFixupsAlloc;
1415 /** Buffer used by the recompiler for recording fixups when generating code. */
1416 PIEMNATIVEFIXUP paFixups;
1417
1418#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1419 /** Actual number of fixups in paTbExitFixups. */
1420 uint32_t cTbExitFixups;
1421 /** Max number of entries allowed in paTbExitFixups before reallocating it. */
1422 uint32_t cTbExitFixupsAlloc;
1423 /** Buffer used by the recompiler for recording fixups when generating code. */
1424 PIEMNATIVEEXITFIXUP paTbExitFixups;
1425#endif
1426
1427#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1428 /** Number of debug info entries allocated for pDbgInfo. */
1429 uint32_t cDbgInfoAlloc;
1430 uint32_t uPadding;
1431 /** Debug info. */
1432 PIEMTBDBG pDbgInfo;
1433#endif
1434
1435#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
1436 /** The current call index (liveness array and threaded calls in TB). */
1437 uint32_t idxCurCall;
1438 /** Number of liveness entries allocated. */
1439 uint32_t cLivenessEntriesAlloc;
1440 /** Liveness entries for all the calls in the TB begin recompiled.
1441 * The entry for idxCurCall contains the info for what the next call will
1442 * require wrt registers. (Which means the last entry is the initial liveness
1443 * state.) */
1444 PIEMLIVENESSENTRY paLivenessEntries;
1445#endif
1446
1447 /** The translation block being recompiled. */
1448 PCIEMTB pTbOrg;
1449 /** The VMCPU structure of the EMT. */
1450 PVMCPUCC pVCpu;
1451
1452 /** Condition sequence number (for generating unique labels). */
1453 uint16_t uCondSeqNo;
1454 /** Check IRQ sequence number (for generating unique labels). */
1455 uint16_t uCheckIrqSeqNo;
1456 /** TLB load sequence number (for generating unique labels). */
1457 uint16_t uTlbSeqNo;
1458 /** The current condition stack depth (aCondStack). */
1459 uint8_t cCondDepth;
1460
1461 /** The argument count + hidden regs from the IEM_MC_BEGIN_EX statement. */
1462 uint8_t cArgsX;
1463 /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
1464 uint32_t fCImpl;
1465 /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
1466 uint32_t fMc;
1467 /** The expected IEMCPU::fExec value for the current call/instruction. */
1468 uint32_t fExec;
1469#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1470 /** IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags for exception flags
1471 * we only emit once per TB (or when the cr0/cr4/xcr0 register changes).
1472 *
1473 * This is an optimization because these control registers can only be changed from
1474 * by calling a C helper we can catch. Should reduce the number of instructions in a TB
1475 * consisting of multiple SIMD instructions.
1476 */
1477 uint32_t fSimdRaiseXcptChecksEmitted;
1478#endif
1479 /** The call number of the last CheckIrq, UINT32_MAX if not seen. */
1480 uint32_t idxLastCheckIrqCallNo;
1481
1482 /** Core state requiring care with branches. */
1483 IEMNATIVECORESTATE Core;
1484
1485 /** The condition nesting stack. */
1486 IEMNATIVECOND aCondStack[2];
1487
1488#ifndef IEM_WITH_THROW_CATCH
1489 /** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
1490 * for recompilation error handling. */
1491 jmp_buf JmpBuf;
1492#endif
1493} IEMRECOMPILERSTATE;
1494/** Pointer to a native recompiler state. */
1495typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
1496
1497
1498/** @def IEMNATIVE_TRY_SETJMP
1499 * Wrapper around setjmp / try, hiding all the ugly differences.
1500 *
1501 * @note Use with extreme care as this is a fragile macro.
1502 * @param a_pReNative The native recompile state.
1503 * @param a_rcTarget The variable that should receive the status code in case
1504 * of a longjmp/throw.
1505 */
1506/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
1507 * Start wrapper for catch / setjmp-else.
1508 *
1509 * This will set up a scope.
1510 *
1511 * @note Use with extreme care as this is a fragile macro.
1512 * @param a_pReNative The native recompile state.
1513 * @param a_rcTarget The variable that should receive the status code in case
1514 * of a longjmp/throw.
1515 */
1516/** @def IEMNATIVE_CATCH_LONGJMP_END
1517 * End wrapper for catch / setjmp-else.
1518 *
1519 * This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
1520 * up the state.
1521 *
1522 * @note Use with extreme care as this is a fragile macro.
1523 * @param a_pReNative The native recompile state.
1524 */
1525/** @def IEMNATIVE_DO_LONGJMP
1526 *
1527 * Wrapper around longjmp / throw.
1528 *
1529 * @param a_pReNative The native recompile state.
1530 * @param a_rc The status code jump back with / throw.
1531 */
1532#ifdef IEM_WITH_THROW_CATCH
1533# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1534 a_rcTarget = VINF_SUCCESS; \
1535 try
1536# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1537 catch (int rcThrown) \
1538 { \
1539 a_rcTarget = rcThrown
1540# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1541 } \
1542 ((void)0)
1543# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
1544#else /* !IEM_WITH_THROW_CATCH */
1545# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
1546 if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
1547# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
1548 else \
1549 { \
1550 ((void)0)
1551# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
1552 }
1553# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
1554#endif /* !IEM_WITH_THROW_CATCH */
1555
1556
1557/**
1558 * Native recompiler worker for a threaded function.
1559 *
1560 * @returns New code buffer offset; throws VBox status code in case of a failure.
1561 * @param pReNative The native recompiler state.
1562 * @param off The current code buffer offset.
1563 * @param pCallEntry The threaded call entry.
1564 *
1565 * @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
1566 */
1567typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
1568/** Pointer to a native recompiler worker for a threaded function. */
1569typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
1570
1571/** Defines a native recompiler worker for a threaded function.
1572 * @see FNIEMNATIVERECOMPFUNC */
1573#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
1574 uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
1575
1576/** Prototypes a native recompiler function for a threaded function.
1577 * @see FNIEMNATIVERECOMPFUNC */
1578#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1579
1580
1581/**
1582 * Native recompiler liveness analysis worker for a threaded function.
1583 *
1584 * @param pCallEntry The threaded call entry.
1585 * @param pIncoming The incoming liveness state entry.
1586 * @param pOutgoing The outgoing liveness state entry.
1587 */
1588typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1589 PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1590/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1591typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1592
1593/** Defines a native recompiler liveness analysis worker for a threaded function.
1594 * @see FNIEMNATIVELIVENESSFUNC */
1595#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1596 DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1597
1598/** Prototypes a native recompiler liveness analysis function for a threaded function.
1599 * @see FNIEMNATIVELIVENESSFUNC */
1600#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1601
1602
1603/** Define a native recompiler helper function, safe to call from the TB code. */
1604#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1605 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1606/** Prototype a native recompiler helper function, safe to call from the TB code. */
1607#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1608 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1609/** Pointer typedef a native recompiler helper function, safe to call from the TB code. */
1610#define IEM_DECL_NATIVE_HLP_PTR(a_RetType, a_Name, a_ArgList) \
1611 a_RetType (VBOXCALL *a_Name) a_ArgList
1612
1613
1614#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
1615DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddNativeOffset(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1616DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegShadowing(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg,
1617 uint8_t idxHstReg = UINT8_MAX, uint8_t idxHstRegPrev = UINT8_MAX);
1618# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1619DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestSimdRegShadowing(PIEMRECOMPILERSTATE pReNative,
1620 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1621 uint8_t idxHstSimdReg = UINT8_MAX,
1622 uint8_t idxHstSimdRegPrev = UINT8_MAX);
1623# endif
1624# if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) || defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
1625DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegDirty(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1626 uint8_t idxGstReg, uint8_t idxHstReg);
1627DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddGuestRegWriteback(PIEMRECOMPILERSTATE pReNative, bool fSimdReg,
1628 uint64_t fGstReg);
1629# endif
1630DECL_HIDDEN_THROW(void) iemNativeDbgInfoAddDelayedPcUpdate(PIEMRECOMPILERSTATE pReNative,
1631 uint32_t offPc, uint32_t cInstrSkipped);
1632#endif /* IEMNATIVE_WITH_TB_DEBUG_INFO */
1633
1634DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1635 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1636DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1637DECLHIDDEN(uint32_t) iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1638 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
1639DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1640 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1641#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
1642DECL_HIDDEN_THROW(void) iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVELABELTYPE enmExitReason);
1643#endif
1644DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1645
1646DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1647DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1648 bool fPreferVolatile = true);
1649DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1650 bool fPreferVolatile = true);
1651DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1652 IEMNATIVEGSTREG enmGstReg,
1653 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1654 bool fNoVolatileRegs = false, bool fSkipLivenessAssert = false);
1655DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1656 IEMNATIVEGSTREG enmGstReg);
1657
1658DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1659DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1660#if (defined(IPRT_INCLUDED_x86_h) && defined(RT_ARCH_AMD64)) || (defined(IPRT_INCLUDED_armv8_h) && defined(RT_ARCH_ARM64))
1661DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1662 uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_GREG_MASK);
1663# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1664DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegMoveOrSpillStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar,
1665 uint32_t fForbiddenRegs = IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK);
1666# endif
1667#endif
1668DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1669DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1670DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1671DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1672#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1673DECLHIDDEN(void) iemNativeSimdRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, bool fFlushShadows) RT_NOEXCEPT;
1674# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1675DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushDirtyGuestByHostSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1676# endif
1677#endif
1678DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1679DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1680 uint32_t fKeepVars = 0);
1681DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1682DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1683DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1684 uint32_t fHstRegsActiveShadows);
1685#ifdef VBOX_STRICT
1686DECLHIDDEN(void) iemNativeRegAssertSanity(PIEMRECOMPILERSTATE pReNative);
1687#endif
1688DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWritesSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept,
1689 uint64_t fGstSimdShwExcept);
1690#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1691DECL_HIDDEN_THROW(uint32_t) iemNativeEmitPcWritebackSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1692#endif
1693#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
1694DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREG enmGstReg);
1695DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuest(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fFlushGstReg = UINT64_MAX);
1696DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushDirtyGuestByHostRegShadow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxHstReg);
1697#endif
1698
1699
1700#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1701DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1702DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1703 bool fPreferVolatile = true);
1704DECL_HIDDEN_THROW(uint8_t) iemNativeSimdRegAllocTmpForGuestSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1705 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1706 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz,
1707 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1708 bool fNoVolatileRegs = false);
1709DECLHIDDEN(void) iemNativeSimdRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg) RT_NOEXCEPT;
1710DECLHIDDEN(void) iemNativeSimdRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstSimdRegs) RT_NOEXCEPT;
1711DECL_HIDDEN_THROW(uint32_t) iemNativeSimdRegFlushPendingWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1712 IEMNATIVEGSTSIMDREG enmGstSimdReg);
1713DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadSimdRegWithGstShadowSimdReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1714 uint8_t idxHstSimdReg, IEMNATIVEGSTSIMDREG enmGstSimdReg,
1715 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1716#endif
1717
1718DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType);
1719DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue);
1720DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar);
1721DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType);
1722DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue);
1723DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocAssign(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t cbType, uint8_t idxVarOther);
1724DECL_HIDDEN_THROW(void) iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1725DECL_HIDDEN_THROW(void) iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue);
1726DECL_HIDDEN_THROW(void) iemNativeVarSetKindToGstRegRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1727 IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg);
1728DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1729DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1730 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1731#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1732DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1733 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1734#endif
1735DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1736 IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1737DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1738 uint32_t fHstRegsNotToSave);
1739DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1740 uint32_t fHstRegsNotToSave);
1741DECLHIDDEN(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1742DECLHIDDEN(void) iemNativeVarFreeAllSlow(PIEMRECOMPILERSTATE pReNative, uint32_t bmVars);
1743
1744DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1745 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1746#ifdef VBOX_STRICT
1747DECL_HIDDEN_THROW(uint32_t) iemNativeEmitTop32BitsClearCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg);
1748DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg,
1749 IEMNATIVEGSTREG enmGstReg);
1750# ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1751DECL_HIDDEN_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxSimdReg,
1752 IEMNATIVEGSTSIMDREG enmGstSimdReg,
1753 IEMNATIVEGSTSIMDREGLDSTSZ enmLoadSz);
1754# endif
1755DECL_HIDDEN_THROW(uint32_t) iemNativeEmitExecFlagsCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fExec);
1756#endif
1757#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
1758DECL_HIDDEN_THROW(uint32_t) iemNativeEmitEFlagsSkippingCheck(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflNeeded);
1759#endif
1760DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1761DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs, bool fFlushPendingWrites = true);
1762DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1763 uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1764 uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1765DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1766 PCIEMTHRDEDCALLENTRY pCallEntry);
1767DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGprCanonicalMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1768 uint8_t idxAddrReg, uint8_t idxInstr);
1769DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckGpr32AgainstCsSegLimitMaybeRaiseGp0(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1770 uint8_t idxAddrReg, uint8_t idxInstr);
1771DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
1772 IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass);
1773
1774
1775IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecStatusCodeFiddling,(PVMCPUCC pVCpu, int rc, uint8_t idxInstr));
1776IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseGp0,(PVMCPUCC pVCpu));
1777IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseNm,(PVMCPUCC pVCpu));
1778IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseUd,(PVMCPUCC pVCpu));
1779IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseMf,(PVMCPUCC pVCpu));
1780IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseXf,(PVMCPUCC pVCpu));
1781IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseDe,(PVMCPUCC pVCpu));
1782IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpObsoleteTb,(PVMCPUCC pVCpu));
1783IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
1784IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
1785IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
1786IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
1787IEM_DECL_NATIVE_HLP_PROTO(int, iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
1788
1789IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1790IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1791IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1792IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1793IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1794IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1795IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1796IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1797IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1798IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
1799#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1800IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1801IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1802IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT128U pu128Dst));
1803IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1804IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PRTUINT256U pu256Dst));
1805#endif
1806IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value));
1807IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint16_t u16Value));
1808IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint32_t u32Value));
1809IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint64_t u64Value));
1810#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1811IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1812IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT128U pu128Src));
1813IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1814IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, PCRTUINT256U pu256Src));
1815#endif
1816IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1817IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1818IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1819IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1820IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1821IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1822IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1823
1824IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1825IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1826IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1827IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU8_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1828IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1829IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1830IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU16_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1831IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1832IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU32_Sx_U64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1833IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFlatFetchDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1834#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1835IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1836IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1837IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT128U pu128Dst));
1838IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1839IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatFetchDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PRTUINT256U pu256Dst));
1840#endif
1841IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t u8Value));
1842IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1843IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1844IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1845#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
1846IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128AlignedSse,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1847IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU128NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT128U pu128Src));
1848IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256NoAc,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1849IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemFlatStoreDataU256AlignedAvx,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, PCRTUINT256U pu256Src));
1850#endif
1851IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value));
1852IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1853IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value));
1854IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value));
1855IEM_DECL_NATIVE_HLP_PROTO(uint16_t, iemNativeHlpStackFlatFetchU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1856IEM_DECL_NATIVE_HLP_PROTO(uint32_t, iemNativeHlpStackFlatFetchU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1857IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpStackFlatFetchU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem));
1858
1859IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1860IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1861IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1862IEM_DECL_NATIVE_HLP_PROTO(uint8_t const *, iemNativeHlpMemMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1863IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1864IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1865IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1866IEM_DECL_NATIVE_HLP_PROTO(uint16_t const *, iemNativeHlpMemMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1867IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1868IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1869IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1870IEM_DECL_NATIVE_HLP_PROTO(uint32_t const *, iemNativeHlpMemMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1871IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1872IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1873IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1874IEM_DECL_NATIVE_HLP_PROTO(uint64_t const *, iemNativeHlpMemMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1875IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U *, iemNativeHlpMemMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1876IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U *, iemNativeHlpMemMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1877IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1878IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1879IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1880IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const *, iemNativeHlpMemMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem, uint8_t iSegReg));
1881
1882IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1883IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1884IEM_DECL_NATIVE_HLP_PROTO(uint8_t *, iemNativeHlpMemFlatMapDataU8Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1885IEM_DECL_NATIVE_HLP_PROTO(uint8_t const *, iemNativeHlpMemFlatMapDataU8Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1886IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1887IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1888IEM_DECL_NATIVE_HLP_PROTO(uint16_t *, iemNativeHlpMemFlatMapDataU16Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1889IEM_DECL_NATIVE_HLP_PROTO(uint16_t const *, iemNativeHlpMemFlatMapDataU16Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1890IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1891IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1892IEM_DECL_NATIVE_HLP_PROTO(uint32_t *, iemNativeHlpMemFlatMapDataU32Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1893IEM_DECL_NATIVE_HLP_PROTO(uint32_t const *, iemNativeHlpMemFlatMapDataU32Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1894IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1895IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1896IEM_DECL_NATIVE_HLP_PROTO(uint64_t *, iemNativeHlpMemFlatMapDataU64Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1897IEM_DECL_NATIVE_HLP_PROTO(uint64_t const *, iemNativeHlpMemFlatMapDataU64Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1898IEM_DECL_NATIVE_HLP_PROTO(RTFLOAT80U *, iemNativeHlpMemFlatMapDataR80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1899IEM_DECL_NATIVE_HLP_PROTO(RTPBCD80U *, iemNativeHlpMemFlatMapDataD80Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1900IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1901IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Rw,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1902IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Wo,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1903IEM_DECL_NATIVE_HLP_PROTO(RTUINT128U const *, iemNativeHlpMemFlatMapDataU128Ro,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem));
1904
1905IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1906IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRw,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1907IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapWo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1908IEM_DECL_NATIVE_HLP_PROTO(void, iemNativeHlpMemCommitAndUnmapRo,(PVMCPUCC pVCpu, uint8_t bUnmapInfo));
1909
1910
1911/**
1912 * Info about shadowed guest register values.
1913 * @see IEMNATIVEGSTREG
1914 */
1915typedef struct IEMANTIVEGSTREGINFO
1916{
1917 /** Offset in VMCPU. */
1918 uint32_t off;
1919 /** The field size. */
1920 uint8_t cb;
1921 /** Name (for logging). */
1922 const char *pszName;
1923} IEMANTIVEGSTREGINFO;
1924extern DECL_HIDDEN_DATA(IEMANTIVEGSTREGINFO const) g_aGstShadowInfo[];
1925extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1926extern DECL_HIDDEN_DATA(int32_t const) g_aoffIemNativeCallStackArgBpDisp[];
1927extern DECL_HIDDEN_DATA(uint32_t const) g_afIemNativeCallRegs[];
1928extern DECL_HIDDEN_DATA(uint8_t const) g_aidxIemNativeCallRegs[];
1929
1930
1931
1932/**
1933 * Ensures that there is sufficient space in the instruction output buffer.
1934 *
1935 * This will reallocate the buffer if needed and allowed.
1936 *
1937 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1938 * allocation size.
1939 *
1940 * @returns Pointer to the instruction output buffer on success; throws VBox
1941 * status code on failure, so no need to check it.
1942 * @param pReNative The native recompile state.
1943 * @param off Current instruction offset. Works safely for UINT32_MAX
1944 * as well.
1945 * @param cInstrReq Number of instruction about to be added. It's okay to
1946 * overestimate this a bit.
1947 */
1948DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1949iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1950{
1951 uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1952 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1953 {
1954#ifdef VBOX_STRICT
1955 pReNative->offInstrBufChecked = offChecked;
1956#endif
1957 return pReNative->pInstrBuf;
1958 }
1959 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1960}
1961
1962/**
1963 * Checks that we didn't exceed the space requested in the last
1964 * iemNativeInstrBufEnsure() call.
1965 */
1966#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1967 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1968 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1969
1970/**
1971 * Checks that a variable index is valid.
1972 */
1973#ifdef IEMNATIVE_VAR_IDX_MAGIC
1974# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1975 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1976 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1977 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
1978 ("%s=%#x\n", #a_idxVar, a_idxVar))
1979#else
1980# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1981 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1982 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1983#endif
1984
1985/**
1986 * Checks that a variable index is valid and that the variable is assigned the
1987 * correct argument number.
1988 * This also adds a RT_NOREF of a_idxVar.
1989 */
1990#ifdef IEMNATIVE_VAR_IDX_MAGIC
1991# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1992 RT_NOREF_PV(a_idxVar); \
1993 AssertMsg( ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
1994 && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1995 && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
1996 && (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
1997 ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1998 (a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
1999 RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
2000 a_uArgNo)); \
2001 } while (0)
2002#else
2003# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
2004 RT_NOREF_PV(a_idxVar); \
2005 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
2006 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
2007 && (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
2008 , ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
2009 (a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
2010 } while (0)
2011#endif
2012
2013
2014/**
2015 * Checks that a variable has the expected size.
2016 */
2017#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
2018 AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
2019 ("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
2020 (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar, (a_cbVar)))
2021
2022
2023/**
2024 * Calculates the stack address of a variable as a [r]BP displacement value.
2025 */
2026DECL_FORCE_INLINE(int32_t)
2027iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
2028{
2029 Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
2030 return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
2031}
2032
2033
2034/**
2035 * Releases the variable's register.
2036 *
2037 * The register must have been previously acquired calling
2038 * iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
2039 * iemNativeVarRegisterSetAndAcquire().
2040 */
2041DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2042{
2043 IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
2044 Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
2045 pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
2046}
2047
2048
2049#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2050DECL_INLINE_THROW(void) iemNativeVarSimdRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
2051{
2052 Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
2053 iemNativeVarRegisterRelease(pReNative, idxVar);
2054}
2055#endif
2056
2057
2058/**
2059 * Converts IEM_CIMPL_F_XXX flags into a guest register shadow copy flush mask.
2060 *
2061 * @returns The flush mask.
2062 * @param fCImpl The IEM_CIMPL_F_XXX flags.
2063 * @param fGstShwFlush The starting flush mask.
2064 */
2065DECL_FORCE_INLINE(uint64_t) iemNativeCImplFlagsToGuestShadowFlushMask(uint32_t fCImpl, uint64_t fGstShwFlush)
2066{
2067 if (fCImpl & IEM_CIMPL_F_BRANCH_FAR)
2068 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_CS)
2069 | RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_CS)
2070 | RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_CS);
2071 if (fCImpl & IEM_CIMPL_F_BRANCH_STACK_FAR)
2072 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP)
2073 | RT_BIT_64(kIemNativeGstReg_SegSelFirst + X86_SREG_SS)
2074 | RT_BIT_64(kIemNativeGstReg_SegBaseFirst + X86_SREG_SS)
2075 | RT_BIT_64(kIemNativeGstReg_SegLimitFirst + X86_SREG_SS);
2076 else if (fCImpl & IEM_CIMPL_F_BRANCH_STACK)
2077 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xSP);
2078 if (fCImpl & (IEM_CIMPL_F_RFLAGS | IEM_CIMPL_F_STATUS_FLAGS | IEM_CIMPL_F_INHIBIT_SHADOW))
2079 fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_EFlags);
2080 return fGstShwFlush;
2081}
2082
2083
2084/** Number of hidden arguments for CIMPL calls.
2085 * @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
2086#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
2087# define IEM_CIMPL_HIDDEN_ARGS 3
2088#else
2089# define IEM_CIMPL_HIDDEN_ARGS 2
2090#endif
2091
2092
2093#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2094/** Number of hidden arguments for SSE_AIMPL calls. */
2095# define IEM_SSE_AIMPL_HIDDEN_ARGS 1
2096/** Number of hidden arguments for AVX_AIMPL calls. */
2097# define IEM_AVX_AIMPL_HIDDEN_ARGS 1
2098#endif
2099
2100
2101#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
2102
2103# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2104/**
2105 * Helper for iemNativeLivenessGetStateByGstReg.
2106 *
2107 * @returns IEMLIVENESS_STATE_XXX
2108 * @param fMergedStateExp2 This is the RT_BIT_32() of each sub-state
2109 * ORed together.
2110 */
2111DECL_FORCE_INLINE(uint32_t)
2112iemNativeLivenessMergeExpandedEFlagsState(uint32_t fMergedStateExp2)
2113{
2114 /* INPUT trumps anything else. */
2115 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_INPUT))
2116 return IEMLIVENESS_STATE_INPUT;
2117
2118 /* CLOBBERED trumps XCPT_OR_CALL and UNUSED. */
2119 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_CLOBBERED))
2120 {
2121 /* If not all sub-fields are clobbered they must be considered INPUT. */
2122 if (fMergedStateExp2 & (RT_BIT_32(IEMLIVENESS_STATE_UNUSED) | RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL)))
2123 return IEMLIVENESS_STATE_INPUT;
2124 return IEMLIVENESS_STATE_CLOBBERED;
2125 }
2126
2127 /* XCPT_OR_CALL trumps UNUSED. */
2128 if (fMergedStateExp2 & RT_BIT_32(IEMLIVENESS_STATE_XCPT_OR_CALL))
2129 return IEMLIVENESS_STATE_XCPT_OR_CALL;
2130
2131 return IEMLIVENESS_STATE_UNUSED;
2132}
2133# endif /* !IEMLIVENESS_EXTENDED_LAYOUT */
2134
2135
2136DECL_FORCE_INLINE(uint32_t)
2137iemNativeLivenessGetStateByGstRegEx(PCIEMLIVENESSENTRY pLivenessEntry, unsigned enmGstRegEx)
2138{
2139# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2140 return ((pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2141 | (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2);
2142# else
2143 return ( (pLivenessEntry->Bit0.bm64 >> enmGstRegEx) & 1)
2144 | (((pLivenessEntry->Bit1.bm64 >> enmGstRegEx) << 1) & 2)
2145 | (((pLivenessEntry->Bit2.bm64 >> enmGstRegEx) << 2) & 4)
2146 | (((pLivenessEntry->Bit3.bm64 >> enmGstRegEx) << 2) & 8);
2147# endif
2148}
2149
2150
2151DECL_FORCE_INLINE(uint32_t)
2152iemNativeLivenessGetStateByGstReg(PCIEMLIVENESSENTRY pLivenessEntry, IEMNATIVEGSTREG enmGstReg)
2153{
2154 uint32_t uRet = iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, (unsigned)enmGstReg);
2155 if (enmGstReg == kIemNativeGstReg_EFlags)
2156 {
2157 /* Merge the eflags states to one. */
2158# ifndef IEMLIVENESS_EXTENDED_LAYOUT
2159 uRet = RT_BIT_32(uRet);
2160 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflCf | (pLivenessEntry->Bit1.fEflCf << 1));
2161 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflPf | (pLivenessEntry->Bit1.fEflPf << 1));
2162 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflAf | (pLivenessEntry->Bit1.fEflAf << 1));
2163 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflZf | (pLivenessEntry->Bit1.fEflZf << 1));
2164 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflSf | (pLivenessEntry->Bit1.fEflSf << 1));
2165 uRet |= RT_BIT_32(pLivenessEntry->Bit0.fEflOf | (pLivenessEntry->Bit1.fEflOf << 1));
2166 uRet = iemNativeLivenessMergeExpandedEFlagsState(uRet);
2167# else
2168 AssertCompile(IEMLIVENESSBIT_IDX_EFL_OTHER == (unsigned)kIemNativeGstReg_EFlags);
2169 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_CF);
2170 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_PF);
2171 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_AF);
2172 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_ZF);
2173 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_SF);
2174 uRet |= iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, IEMLIVENESSBIT_IDX_EFL_OF);
2175# endif
2176 }
2177 return uRet;
2178}
2179
2180
2181# ifdef VBOX_STRICT
2182/** For assertions only, user checks that idxCurCall isn't zerow. */
2183DECL_FORCE_INLINE(uint32_t)
2184iemNativeLivenessGetPrevStateByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg)
2185{
2186 return iemNativeLivenessGetStateByGstReg(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], enmGstReg);
2187}
2188# endif /* VBOX_STRICT */
2189
2190#endif /* IEMNATIVE_WITH_LIVENESS_ANALYSIS */
2191
2192
2193/**
2194 * Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
2195 */
2196DECL_FORCE_INLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
2197{
2198 if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
2199 return IEM_CIMPL_HIDDEN_ARGS;
2200 if (pReNative->fCImpl & (IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE | IEM_CIMPL_F_CALLS_AIMPL_WITH_XSTATE))
2201 return 1;
2202 return 0;
2203}
2204
2205
2206DECL_FORCE_INLINE(uint8_t) iemNativeRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, unsigned idxReg,
2207 IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2208{
2209 pReNative->Core.bmHstRegs |= RT_BIT_32(idxReg);
2210
2211 pReNative->Core.aHstRegs[idxReg].enmWhat = enmWhat;
2212 pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
2213 pReNative->Core.aHstRegs[idxReg].idxVar = idxVar;
2214 return (uint8_t)idxReg;
2215}
2216
2217
2218
2219/*********************************************************************************************************************************
2220* Register Allocator (GPR) *
2221*********************************************************************************************************************************/
2222
2223/**
2224 * Marks host register @a idxHstReg as containing a shadow copy of guest
2225 * register @a enmGstReg.
2226 *
2227 * ASSUMES that caller has made sure @a enmGstReg is not associated with any
2228 * host register before calling.
2229 */
2230DECL_FORCE_INLINE(void)
2231iemNativeRegMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2232{
2233 Assert(!(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg)));
2234 Assert(!pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows);
2235 Assert((unsigned)enmGstReg < (unsigned)kIemNativeGstReg_End);
2236
2237 pReNative->Core.aidxGstRegShadows[enmGstReg] = idxHstReg;
2238 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = RT_BIT_64(enmGstReg); /** @todo why? not OR? */
2239 pReNative->Core.bmGstRegShadows |= RT_BIT_64(enmGstReg);
2240 pReNative->Core.bmHstRegsWithGstShadow |= RT_BIT_32(idxHstReg);
2241#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2242 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2243 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxHstReg);
2244#else
2245 RT_NOREF(off);
2246#endif
2247}
2248
2249
2250/**
2251 * Clear any guest register shadow claims from @a idxHstReg.
2252 *
2253 * The register does not need to be shadowing any guest registers.
2254 */
2255DECL_FORCE_INLINE(void)
2256iemNativeRegClearGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, uint32_t off)
2257{
2258 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2259 == pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2260 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2261 Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg))
2262 == RT_BOOL(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows));
2263#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2264 Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2265#endif
2266
2267#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2268 uint64_t fGstRegs = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2269 if (fGstRegs)
2270 {
2271 Assert(fGstRegs < RT_BIT_64(kIemNativeGstReg_End));
2272 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2273 while (fGstRegs)
2274 {
2275 unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2276 fGstRegs &= ~RT_BIT_64(iGstReg);
2277 iemNativeDbgInfoAddGuestRegShadowing(pReNative, (IEMNATIVEGSTREG)iGstReg, UINT8_MAX, idxHstReg);
2278 }
2279 }
2280#else
2281 RT_NOREF(off);
2282#endif
2283
2284 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2285 pReNative->Core.bmGstRegShadows &= ~pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows;
2286 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = 0;
2287}
2288
2289
2290/**
2291 * Clear guest register shadow claim regarding @a enmGstReg from @a idxHstReg
2292 * and global overview flags.
2293 */
2294DECL_FORCE_INLINE(void)
2295iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2296{
2297 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2298 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows)
2299 == pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows
2300 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2301 Assert(pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg));
2302 Assert(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & RT_BIT_64(enmGstReg));
2303 Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxHstReg));
2304#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2305 Assert(!(pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
2306#endif
2307
2308#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2309 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2310 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, UINT8_MAX, idxHstReg);
2311#else
2312 RT_NOREF(off);
2313#endif
2314
2315 uint64_t const fGstRegShadowsNew = pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2316 pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows = fGstRegShadowsNew;
2317 if (!fGstRegShadowsNew)
2318 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxHstReg);
2319 pReNative->Core.bmGstRegShadows &= ~RT_BIT_64(enmGstReg);
2320}
2321
2322
2323#if 0 /* unused */
2324/**
2325 * Clear any guest register shadow claim for @a enmGstReg.
2326 */
2327DECL_FORCE_INLINE(void)
2328iemNativeRegClearGstRegShadowingByGstReg(PIEMRECOMPILERSTATE pReNative, IEMNATIVEGSTREG enmGstReg, uint32_t off)
2329{
2330 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2331 if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2332 {
2333 Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] < RT_ELEMENTS(pReNative->Core.aHstRegs));
2334 iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2335 }
2336}
2337#endif
2338
2339
2340/**
2341 * Clear any guest register shadow claim for @a enmGstReg and mark @a idxHstRegNew
2342 * as the new shadow of it.
2343 *
2344 * Unlike the other guest reg shadow helpers, this does the logging for you.
2345 * However, it is the liveness state is not asserted here, the caller must do
2346 * that.
2347 */
2348DECL_FORCE_INLINE(void)
2349iemNativeRegClearAndMarkAsGstRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstRegNew,
2350 IEMNATIVEGSTREG enmGstReg, uint32_t off)
2351{
2352 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2353 if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
2354 {
2355 uint8_t const idxHstRegOld = pReNative->Core.aidxGstRegShadows[enmGstReg];
2356 Assert(idxHstRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
2357 if (idxHstRegOld == idxHstRegNew)
2358 return;
2359 Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s (from %s)\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2360 g_aGstShadowInfo[enmGstReg].pszName, g_apszIemNativeHstRegNames[idxHstRegOld]));
2361 iemNativeRegClearGstRegShadowingOne(pReNative, pReNative->Core.aidxGstRegShadows[enmGstReg], enmGstReg, off);
2362 }
2363 else
2364 Log12(("iemNativeRegClearAndMarkAsGstRegShadow: %s for guest %s\n", g_apszIemNativeHstRegNames[idxHstRegNew],
2365 g_aGstShadowInfo[enmGstReg].pszName));
2366 iemNativeRegMarkAsGstRegShadow(pReNative, idxHstRegNew, enmGstReg, off);
2367}
2368
2369
2370/**
2371 * Transfers the guest register shadow claims of @a enmGstReg from @a idxRegFrom
2372 * to @a idxRegTo.
2373 */
2374DECL_FORCE_INLINE(void)
2375iemNativeRegTransferGstRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxRegFrom, uint8_t idxRegTo,
2376 IEMNATIVEGSTREG enmGstReg, uint32_t off)
2377{
2378 Assert(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & RT_BIT_64(enmGstReg));
2379 Assert(pReNative->Core.aidxGstRegShadows[enmGstReg] == idxRegFrom);
2380 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows)
2381 == pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows
2382 && pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2383 Assert( (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows)
2384 == pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows);
2385 Assert( RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxRegFrom))
2386 == RT_BOOL(pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows));
2387
2388 uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstReg);
2389 pReNative->Core.aHstRegs[idxRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2390 if (!fGstRegShadowsFrom)
2391 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegFrom);
2392 pReNative->Core.bmHstRegsWithGstShadow |= RT_BIT_32(idxRegTo);
2393 pReNative->Core.aHstRegs[idxRegTo].fGstRegShadows |= RT_BIT_64(enmGstReg);
2394 pReNative->Core.aidxGstRegShadows[enmGstReg] = idxRegTo;
2395#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2396 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2397 iemNativeDbgInfoAddGuestRegShadowing(pReNative, enmGstReg, idxRegTo, idxRegFrom);
2398#else
2399 RT_NOREF(off);
2400#endif
2401}
2402
2403
2404/**
2405 * Flushes any delayed guest register writes.
2406 *
2407 * This must be called prior to calling CImpl functions and any helpers that use
2408 * the guest state (like raising exceptions) and such.
2409 *
2410 * This optimization has not yet been implemented. The first target would be
2411 * RIP updates, since these are the most common ones.
2412 *
2413 * @note This function does not flush any shadowing information for guest registers. This needs to be done by
2414 * the caller if it wishes to do so.
2415 */
2416DECL_INLINE_THROW(uint32_t)
2417iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint64_t fGstShwExcept = 0, uint64_t fGstSimdShwExcept = 0)
2418{
2419#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
2420 uint64_t const bmGstRegShadowDirty = pReNative->Core.bmGstRegShadowDirty & ~fGstShwExcept;
2421#else
2422 uint64_t const bmGstRegShadowDirty = 0;
2423#endif
2424#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2425 uint64_t const bmGstSimdRegShadowDirty = (pReNative->Core.bmGstSimdRegShadowDirtyLo128 | pReNative->Core.bmGstSimdRegShadowDirtyHi128)
2426 & ~fGstSimdShwExcept;
2427#else
2428 uint64_t const bmGstSimdRegShadowDirty = 0;
2429#endif
2430#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2431 uint64_t const fWritebackPc = ~(fGstShwExcept & kIemNativeGstReg_Pc);
2432#else
2433 uint64_t const fWritebackPc = 0;
2434#endif
2435 if (bmGstRegShadowDirty | bmGstSimdRegShadowDirty | fWritebackPc)
2436 return iemNativeRegFlushPendingWritesSlow(pReNative, off, fGstShwExcept, fGstSimdShwExcept);
2437
2438 return off;
2439}
2440
2441
2442
2443/*********************************************************************************************************************************
2444* SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge) *
2445*********************************************************************************************************************************/
2446
2447#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
2448
2449DECL_FORCE_INLINE(uint8_t)
2450iemNativeSimdRegMarkAllocated(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdReg,
2451 IEMNATIVEWHAT enmWhat, uint8_t idxVar = UINT8_MAX) RT_NOEXCEPT
2452{
2453 pReNative->Core.bmHstSimdRegs |= RT_BIT_32(idxSimdReg);
2454
2455 pReNative->Core.aHstSimdRegs[idxSimdReg].enmWhat = enmWhat;
2456 pReNative->Core.aHstSimdRegs[idxSimdReg].idxVar = idxVar;
2457 pReNative->Core.aHstSimdRegs[idxSimdReg].fGstRegShadows = 0;
2458 return idxSimdReg;
2459}
2460
2461
2462/**
2463 * Marks host SIMD register @a idxHstSimdReg as containing a shadow copy of guest
2464 * SIMD register @a enmGstSimdReg.
2465 *
2466 * ASSUMES that caller has made sure @a enmGstSimdReg is not associated with any
2467 * host register before calling.
2468 */
2469DECL_FORCE_INLINE(void)
2470iemNativeSimdRegMarkAsGstSimdRegShadow(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg,
2471 IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2472{
2473 Assert(!(pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(enmGstSimdReg)));
2474 Assert(!pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows);
2475 Assert((unsigned)enmGstSimdReg < (unsigned)kIemNativeGstSimdReg_End);
2476
2477 pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxHstSimdReg;
2478 pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows |= RT_BIT_64(enmGstSimdReg);
2479 pReNative->Core.bmGstSimdRegShadows |= RT_BIT_64(enmGstSimdReg);
2480 pReNative->Core.bmHstSimdRegsWithGstShadow |= RT_BIT_32(idxHstSimdReg);
2481#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2482 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2483 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxHstSimdReg);
2484#else
2485 RT_NOREF(off);
2486#endif
2487}
2488
2489
2490/**
2491 * Transfers the guest SIMD register shadow claims of @a enmGstSimdReg from @a idxSimdRegFrom
2492 * to @a idxSimdRegTo.
2493 */
2494DECL_FORCE_INLINE(void)
2495iemNativeSimdRegTransferGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxSimdRegFrom, uint8_t idxSimdRegTo,
2496 IEMNATIVEGSTSIMDREG enmGstSimdReg, uint32_t off)
2497{
2498 Assert(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & RT_BIT_64(enmGstSimdReg));
2499 Assert(pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] == idxSimdRegFrom);
2500 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows)
2501 == pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows
2502 && pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstReg_End));
2503 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows)
2504 == pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows);
2505 Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxSimdRegFrom))
2506 == RT_BOOL(pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows));
2507 Assert( pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded
2508 == pReNative->Core.aHstSimdRegs[idxSimdRegTo].enmLoaded);
2509
2510 uint64_t const fGstRegShadowsFrom = pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows & ~RT_BIT_64(enmGstSimdReg);
2511 pReNative->Core.aHstSimdRegs[idxSimdRegFrom].fGstRegShadows = fGstRegShadowsFrom;
2512 if (!fGstRegShadowsFrom)
2513 {
2514 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxSimdRegFrom);
2515 pReNative->Core.aHstSimdRegs[idxSimdRegFrom].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2516 }
2517 pReNative->Core.bmHstSimdRegsWithGstShadow |= RT_BIT_32(idxSimdRegTo);
2518 pReNative->Core.aHstSimdRegs[idxSimdRegTo].fGstRegShadows |= RT_BIT_64(enmGstSimdReg);
2519 pReNative->Core.aidxGstSimdRegShadows[enmGstSimdReg] = idxSimdRegTo;
2520#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2521 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2522 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, enmGstSimdReg, idxSimdRegTo, idxSimdRegFrom);
2523#else
2524 RT_NOREF(off);
2525#endif
2526}
2527
2528
2529/**
2530 * Clear any guest register shadow claims from @a idxHstSimdReg.
2531 *
2532 * The register does not need to be shadowing any guest registers.
2533 */
2534DECL_FORCE_INLINE(void)
2535iemNativeSimdRegClearGstSimdRegShadowing(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstSimdReg, uint32_t off)
2536{
2537 Assert( (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows)
2538 == pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows
2539 && pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
2540 Assert( RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxHstSimdReg))
2541 == RT_BOOL(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows));
2542 Assert( !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyLo128)
2543 && !(pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows & pReNative->Core.bmGstSimdRegShadowDirtyHi128));
2544
2545#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
2546 uint64_t fGstRegs = pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2547 if (fGstRegs)
2548 {
2549 Assert(fGstRegs < RT_BIT_64(kIemNativeGstSimdReg_End));
2550 iemNativeDbgInfoAddNativeOffset(pReNative, off);
2551 while (fGstRegs)
2552 {
2553 unsigned const iGstReg = ASMBitFirstSetU64(fGstRegs) - 1;
2554 fGstRegs &= ~RT_BIT_64(iGstReg);
2555 iemNativeDbgInfoAddGuestSimdRegShadowing(pReNative, (IEMNATIVEGSTSIMDREG)iGstReg, UINT8_MAX, idxHstSimdReg);
2556 }
2557 }
2558#else
2559 RT_NOREF(off);
2560#endif
2561
2562 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxHstSimdReg);
2563 pReNative->Core.bmGstSimdRegShadows &= ~pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows;
2564 pReNative->Core.aHstSimdRegs[idxHstSimdReg].fGstRegShadows = 0;
2565 pReNative->Core.aHstSimdRegs[idxHstSimdReg].enmLoaded = kIemNativeGstSimdRegLdStSz_Invalid;
2566}
2567
2568#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
2569
2570
2571#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
2572/**
2573 * Emits code to update the guest RIP value by adding the current offset since the start of the last RIP update.
2574 */
2575DECL_INLINE_THROW(uint32_t) iemNativeEmitPcWriteback(PIEMRECOMPILERSTATE pReNative, uint32_t off)
2576{
2577 if (pReNative->Core.offPc)
2578 return iemNativeEmitPcWritebackSlow(pReNative, off);
2579 return off;
2580}
2581#endif /* IEMNATIVE_WITH_DELAYED_PC_UPDATING */
2582
2583
2584#ifdef IEMNATIVE_WITH_RECOMPILER_PROLOGUE_SINGLETON
2585/** @note iemNativeTbEntry returns VBOXSTRICTRC, but we don't declare it as
2586 * it saves us the trouble of a hidden parameter on MSC/amd64. */
2587# ifdef RT_ARCH_AMD64
2588extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, uintptr_t pfnTbBody));
2589# elif defined(RT_ARCH_ARM64)
2590extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEntry, (PVMCPUCC pVCpu, PCPUMCTX pCpumCtx, uintptr_t pfnTbBody));
2591# endif
2592#endif
2593
2594#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
2595extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeFpCtrlRegRestore, (uint64_t u64RegFpCtrl));
2596#endif
2597
2598#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
2599
2600/** @} */
2601
2602#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
2603
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette