IEMN8veRecompiler.h@ 101521

Last change on this file since 101521 was 101518, checked in by vboxsync, 14 months ago
VMM/IEM: Deal with unconditional relative jumps (sans flag checking). [fixes] bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 70.3 KB

Line
1	/* $Id: IEMN8veRecompiler.h 101518 2023-10-20 13:25:33Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler Internals.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36	* @ingroup grp_iem_int
37	* @{
38	*/
39
40	/** @name Stack Frame Layout
41	*
42	* @{ */
43	/** The size of the area for stack variables and spills and stuff.
44	* @note This limit is duplicated in the python script(s). */
45	#define IEMNATIVE_FRAME_VAR_SIZE 0xc0
46	#ifdef RT_ARCH_AMD64
47	/** Number of stack arguments slots for calls made from the frame. */
48	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
49	/** An stack alignment adjustment (between non-volatile register pushes and
50	* the stack variable area, so the latter better aligned). */
51	# define IEMNATIVE_FRAME_ALIGN_SIZE 8
52	/** Number of any shadow arguments (spill area) for calls we make. */
53	# ifdef RT_OS_WINDOWS
54	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
55	# else
56	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
57	# endif
58
59	/** Frame pointer (RBP) relative offset of the last push. */
60	# ifdef RT_OS_WINDOWS
61	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
62	# else
63	# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
64	# endif
65	/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
66	* address for it). */
67	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
68	/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
69	# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
70	/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
71	# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
72	/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
73	# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
74	/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
75	# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
76
77	# ifdef RT_OS_WINDOWS
78	/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
79	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
80	/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
81	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
82	/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
83	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
84	/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
85	# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
86	# endif
87
88	#elif RT_ARCH_ARM64
89	/** No stack argument slots, enough got 8 registers for arguments. */
90	# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
91	/** There are no argument spill area. */
92	# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
93
94	/** Number of saved registers at the top of our stack frame.
95	* This includes the return address and old frame pointer, so x19 thru x30. */
96	# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
97	/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
98	# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
99
100	/** Frame pointer (BP) relative offset of the last push. */
101	# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
102
103	/** Frame pointer (BP) relative offset of the stack variable area (the lowest
104	* address for it). */
105	# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
106
107	#else
108	# error "port me"
109	#endif
110	/** @} */
111
112
113	/** @name Fixed Register Allocation(s)
114	* @{ */
115	/** @def IEMNATIVE_REG_FIXED_PVMCPU
116	* The number of the register holding the pVCpu pointer. */
117	/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
118	* The number of the register holding the &pVCpu->cpum.GstCtx pointer.
119	* @note This not available on AMD64, only ARM64. */
120	/** @def IEMNATIVE_REG_FIXED_TMP0
121	* Dedicated temporary register.
122	* @todo replace this by a register allocator and content tracker. */
123	/** @def IEMNATIVE_REG_FIXED_MASK
124	* Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
125	* architecture. */
126	#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
127	# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
128	# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
129	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
130	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
131	\| RT_BIT_32(X86_GREG_xSP) \
132	\| RT_BIT_32(X86_GREG_xBP) )
133
134	#elif defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
135	# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
136	# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
137	# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
138	# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
139	\| RT_BIT_32(ARMV8_A64_REG_LR) \
140	\| RT_BIT_32(ARMV8_A64_REG_BP) \
141	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
142	\| RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
143	\| RT_BIT_32(ARMV8_A64_REG_X18) \
144	\| RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
145
146	#else
147	# error "port me"
148	#endif
149	/** @} */
150
151	/** @name Call related registers.
152	* @{ */
153	/** @def IEMNATIVE_CALL_RET_GREG
154	* The return value register. */
155	/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
156	* Number of arguments in registers. */
157	/** @def IEMNATIVE_CALL_ARG0_GREG
158	* The general purpose register carrying argument \#0. */
159	/** @def IEMNATIVE_CALL_ARG1_GREG
160	* The general purpose register carrying argument \#1. */
161	/** @def IEMNATIVE_CALL_ARG2_GREG
162	* The general purpose register carrying argument \#2. */
163	/** @def IEMNATIVE_CALL_ARG3_GREG
164	* The general purpose register carrying argument \#3. */
165	/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
166	* Mask of registers the callee will not save and may trash. */
167	#ifdef RT_ARCH_AMD64
168	# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
169
170	# ifdef RT_OS_WINDOWS
171	# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
172	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
173	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
174	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
175	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
176	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
177	\| RT_BIT_32(X86_GREG_xCX) \
178	\| RT_BIT_32(X86_GREG_xDX) \
179	\| RT_BIT_32(X86_GREG_x8) \
180	\| RT_BIT_32(X86_GREG_x9) \
181	\| RT_BIT_32(X86_GREG_x10) \
182	\| RT_BIT_32(X86_GREG_x11) )
183	# else
184	# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
185	# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
186	# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
187	# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
188	# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
189	# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
190	# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
191	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
192	\| RT_BIT_32(X86_GREG_xCX) \
193	\| RT_BIT_32(X86_GREG_xDX) \
194	\| RT_BIT_32(X86_GREG_xDI) \
195	\| RT_BIT_32(X86_GREG_xSI) \
196	\| RT_BIT_32(X86_GREG_x8) \
197	\| RT_BIT_32(X86_GREG_x9) \
198	\| RT_BIT_32(X86_GREG_x10) \
199	\| RT_BIT_32(X86_GREG_x11) )
200	# endif
201
202	#elif defined(RT_ARCH_ARM64)
203	# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
204	# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
205	# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
206	# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
207	# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
208	# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
209	# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
210	# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
211	# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
212	# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
213	# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
214	\| RT_BIT_32(ARMV8_A64_REG_X1) \
215	\| RT_BIT_32(ARMV8_A64_REG_X2) \
216	\| RT_BIT_32(ARMV8_A64_REG_X3) \
217	\| RT_BIT_32(ARMV8_A64_REG_X4) \
218	\| RT_BIT_32(ARMV8_A64_REG_X5) \
219	\| RT_BIT_32(ARMV8_A64_REG_X6) \
220	\| RT_BIT_32(ARMV8_A64_REG_X7) \
221	\| RT_BIT_32(ARMV8_A64_REG_X8) \
222	\| RT_BIT_32(ARMV8_A64_REG_X9) \
223	\| RT_BIT_32(ARMV8_A64_REG_X10) \
224	\| RT_BIT_32(ARMV8_A64_REG_X11) \
225	\| RT_BIT_32(ARMV8_A64_REG_X12) \
226	\| RT_BIT_32(ARMV8_A64_REG_X13) \
227	\| RT_BIT_32(ARMV8_A64_REG_X14) \
228	\| RT_BIT_32(ARMV8_A64_REG_X15) \
229	\| RT_BIT_32(ARMV8_A64_REG_X16) \
230	\| RT_BIT_32(ARMV8_A64_REG_X17) )
231
232	#endif
233
234	/** @} */
235
236
237	/** @def IEMNATIVE_HST_GREG_COUNT
238	* Number of host general purpose registers we tracker. */
239	/** @def IEMNATIVE_HST_GREG_MASK
240	* Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
241	* inverted register masks and such to get down to a correct set of regs. */
242	#ifdef RT_ARCH_AMD64
243	# define IEMNATIVE_HST_GREG_COUNT 16
244	# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
245
246	#elif defined(RT_ARCH_ARM64)
247	# define IEMNATIVE_HST_GREG_COUNT 32
248	# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
249	#else
250	# error "Port me!"
251	#endif
252
253
254	/** Native code generator label types. */
255	typedef enum
256	{
257	kIemNativeLabelType_Invalid = 0,
258	kIemNativeLabelType_Return,
259	kIemNativeLabelType_NonZeroRetOrPassUp,
260	kIemNativeLabelType_RaiseGp0,
261	kIemNativeLabelType_End
262	} IEMNATIVELABELTYPE;
263
264	/** Native code generator label definition. */
265	typedef struct IEMNATIVELABEL
266	{
267	/** Code offset if defined, UINT32_MAX if it needs to be generated after/in
268	* the epilog. */
269	uint32_t off;
270	/** The type of label (IEMNATIVELABELTYPE). */
271	uint16_t enmType;
272	/** Additional label data, type specific. */
273	uint16_t uData;
274	} IEMNATIVELABEL;
275	/** Pointer to a label. */
276	typedef IEMNATIVELABEL *PIEMNATIVELABEL;
277
278
279	/** Native code generator fixup types. */
280	typedef enum
281	{
282	kIemNativeFixupType_Invalid = 0,
283	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
284	/** AMD64 fixup: PC relative 32-bit with addend in bData. */
285	kIemNativeFixupType_Rel32,
286	#elif defined(RT_ARCH_ARM64)
287	/** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ). */
288	kIemNativeFixupType_RelImm19At5,
289	#endif
290	kIemNativeFixupType_End
291	} IEMNATIVEFIXUPTYPE;
292
293	/** Native code generator fixup. */
294	typedef struct IEMNATIVEFIXUP
295	{
296	/** Code offset of the fixup location. */
297	uint32_t off;
298	/** The IEMNATIVELABEL this is a fixup for. */
299	uint16_t idxLabel;
300	/** The fixup type (IEMNATIVEFIXUPTYPE). */
301	uint8_t enmType;
302	/** Addend or other data. */
303	int8_t offAddend;
304	} IEMNATIVEFIXUP;
305	/** Pointer to a native code generator fixup. */
306	typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
307
308
309	/**
310	* Guest registers that can be shadowed in GPRs.
311	*/
312	typedef enum IEMNATIVEGSTREG : uint8_t
313	{
314	kIemNativeGstReg_GprFirst = 0,
315	kIemNativeGstReg_GprLast = 15,
316	kIemNativeGstReg_Pc,
317	kIemNativeGstReg_Rflags,
318	/* gap: 18..23 */
319	kIemNativeGstReg_SegSelFirst = 24,
320	kIemNativeGstReg_SegSelLast = 29,
321	kIemNativeGstReg_SegBaseFirst = 30,
322	kIemNativeGstReg_SegBaseLast = 35,
323	kIemNativeGstReg_SegLimitFirst = 36,
324	kIemNativeGstReg_SegLimitLast = 41,
325	kIemNativeGstReg_End
326	} IEMNATIVEGSTREG;
327
328	/**
329	* Guest registers (classes) that can be referenced.
330	*/
331	typedef enum IEMNATIVEGSTREGREF : uint8_t
332	{
333	kIemNativeGstRegRef_Invalid = 0,
334	kIemNativeGstRegRef_Gpr,
335	kIemNativeGstRegRef_GprHighByte, /*< AH, CH, DH, BH/
336	kIemNativeGstRegRef_EFlags,
337	kIemNativeGstRegRef_MxCsr,
338	kIemNativeGstRegRef_FpuReg,
339	kIemNativeGstRegRef_MReg,
340	kIemNativeGstRegRef_XReg,
341	kIemNativeGstRegRef_YReg,
342	kIemNativeGstRegRef_End
343	} IEMNATIVEGSTREGREF;
344
345
346	/** Variable kinds. */
347	typedef enum IEMNATIVEVARKIND : uint8_t
348	{
349	/** Customary invalid zero value. */
350	kIemNativeVarKind_Invalid = 0,
351	/** This is either in a register or on the stack. */
352	kIemNativeVarKind_Stack,
353	/** Immediate value - loaded into register when needed, or can live on the
354	* stack if referenced (in theory). */
355	kIemNativeVarKind_Immediate,
356	/** Variable reference - loaded into register when needed, never stack. */
357	kIemNativeVarKind_VarRef,
358	/** Guest register reference - loaded into register when needed, never stack. */
359	kIemNativeVarKind_GstRegRef,
360	/** End of valid values. */
361	kIemNativeVarKind_End
362	} IEMNATIVEVARKIND;
363
364
365	/** Variable or argument. */
366	typedef struct IEMNATIVEVAR
367	{
368	/** The kind of variable. */
369	IEMNATIVEVARKIND enmKind;
370	/** The variable size in bytes. */
371	uint8_t cbVar;
372	/** The first stack slot (uint64_t), except for immediate and references
373	* where it usually is UINT8_MAX. */
374	uint8_t idxStackSlot;
375	/** The host register allocated for the variable, UINT8_MAX if not. */
376	uint8_t idxReg;
377	/** The argument number if argument, UINT8_MAX if regular variable. */
378	uint8_t uArgNo;
379	/** If referenced, the index of the variable referencing this one, otherwise
380	* UINT8_MAX. A referenced variable must only be placed on the stack and
381	* must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
382	uint8_t idxReferrerVar;
383	/** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */
384	IEMNATIVEGSTREG enmGstReg;
385	uint8_t bAlign;
386
387	union
388	{
389	/** kIemNativeVarKind_Immediate: The immediate value. */
390	uint64_t uValue;
391	/** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
392	uint8_t idxRefVar;
393	/** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
394	struct
395	{
396	/** The class of register. */
397	IEMNATIVEGSTREGREF enmClass;
398	/** Index within the class. */
399	uint8_t idx;
400	} GstRegRef;
401	} u;
402	} IEMNATIVEVAR;
403
404	/** What is being kept in a host register. */
405	typedef enum IEMNATIVEWHAT : uint8_t
406	{
407	/** The traditional invalid zero value. */
408	kIemNativeWhat_Invalid = 0,
409	/** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
410	kIemNativeWhat_Var,
411	/** Temporary register, this is typically freed when a MC completes. */
412	kIemNativeWhat_Tmp,
413	/** Call argument w/o a variable mapping. This is free (via
414	* IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
415	kIemNativeWhat_Arg,
416	/** Return status code.
417	* @todo not sure if we need this... */
418	kIemNativeWhat_rc,
419	/** The fixed pVCpu (PVMCPUCC) register.
420	* @todo consider offsetting this on amd64 to use negative offsets to access
421	* more members using 8-byte disp. */
422	kIemNativeWhat_pVCpuFixed,
423	/** The fixed pCtx (PCPUMCTX) register.
424	* @todo consider offsetting this on amd64 to use negative offsets to access
425	* more members using 8-byte disp. */
426	kIemNativeWhat_pCtxFixed,
427	/** Fixed temporary register. */
428	kIemNativeWhat_FixedTmp,
429	/** Register reserved by the CPU or OS architecture. */
430	kIemNativeWhat_FixedReserved,
431	/** End of valid values. */
432	kIemNativeWhat_End
433	} IEMNATIVEWHAT;
434
435	/**
436	* Host general register entry.
437	*
438	* The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
439	*
440	* @todo Track immediate values in host registers similarlly to how we track the
441	* guest register shadow copies. For it to be real helpful, though,
442	* we probably need to know which will be reused and put them into
443	* non-volatile registers, otherwise it's going to be more or less
444	* restricted to an instruction or two.
445	*/
446	typedef struct IEMNATIVEHSTREG
447	{
448	/** Set of guest registers this one shadows.
449	*
450	* Using a bitmap here so we can designate the same host register as a copy
451	* for more than one guest register. This is expected to be useful in
452	* situations where one value is copied to several registers in a sequence.
453	* If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
454	* sequence we'd want to let this register follow to be a copy of and there
455	* will always be places where we'd be picking the wrong one.
456	*/
457	uint64_t fGstRegShadows;
458	/** What is being kept in this register. */
459	IEMNATIVEWHAT enmWhat;
460	/** Variable index if holding a variable, otherwise UINT8_MAX. */
461	uint8_t idxVar;
462	/** Alignment padding. */
463	uint8_t abAlign[6];
464	} IEMNATIVEHSTREG;
465
466
467	/**
468	* Native recompiler state.
469	*/
470	typedef struct IEMRECOMPILERSTATE
471	{
472	/** Size of the buffer that pbNativeRecompileBufR3 points to in
473	* IEMNATIVEINSTR units. */
474	uint32_t cInstrBufAlloc;
475	uint32_t uPadding; /* We don't keep track of this here... */
476	/** Fixed temporary code buffer for native recompilation. */
477	PIEMNATIVEINSTR pInstrBuf;
478
479	/** Bitmaps with the label types used. */
480	uint64_t bmLabelTypes;
481	/** Actual number of labels in paLabels. */
482	uint32_t cLabels;
483	/** Max number of entries allowed in paLabels before reallocating it. */
484	uint32_t cLabelsAlloc;
485	/** Labels defined while recompiling (referenced by fixups). */
486	PIEMNATIVELABEL paLabels;
487
488	/** Actual number of fixups paFixups. */
489	uint32_t cFixups;
490	/** Max number of entries allowed in paFixups before reallocating it. */
491	uint32_t cFixupsAlloc;
492	/** Buffer used by the recompiler for recording fixups when generating code. */
493	PIEMNATIVEFIXUP paFixups;
494
495	/** The translation block being recompiled. */
496	PCIEMTB pTbOrg;
497
498	/** Allocation bitmap fro aHstRegs. */
499	uint32_t bmHstRegs;
500
501	/** Bitmap marking which host register contains guest register shadow copies.
502	* This is used during register allocation to try preserve copies. */
503	uint32_t bmHstRegsWithGstShadow;
504	/** Bitmap marking valid entries in aidxGstRegShadows. */
505	uint64_t bmGstRegShadows;
506
507	/** Allocation bitmap for aVars. */
508	uint32_t bmVars;
509	uint32_t u32Align;
510	union
511	{
512	/** Index of variable arguments, UINT8_MAX if not valid. */
513	uint8_t aidxArgVars[8];
514	/** For more efficient resetting. */
515	uint64_t u64ArgVars;
516	};
517
518	/** Host register allocation tracking. */
519	IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
520	/** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
521	* Entries are only valid if the corresponding bit in bmGstRegShadows is set.
522	* (A shadow copy of a guest register can only be held in a one host register,
523	* there are no duplicate copies or ambiguities like that). */
524	uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
525	/** Variables and arguments. */
526	IEMNATIVEVAR aVars[16];
527	} IEMRECOMPILERSTATE;
528	/** Pointer to a native recompiler state. */
529	typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
530
531
532	/**
533	* Native recompiler worker for a threaded function.
534	*
535	* @returns New code buffer offset, UINT32_MAX in case of failure.
536	* @param pReNative The native recompiler state.
537	* @param off The current code buffer offset.
538	* @param pCallEntry The threaded call entry.
539	*
540	* @note This is not allowed to throw anything atm.
541	*/
542	typedef DECLCALLBACKTYPE(uint32_t, FNIEMNATIVERECOMPFUNC,(PIEMRECOMPILERSTATE pReNative, uint32_t off,
543	PCIEMTHRDEDCALLENTRY pCallEntry));
544	/** Pointer to a native recompiler worker for a threaded function. */
545	typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
546
547	/** Defines a native recompiler worker for a threaded function. */
548	#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
549	DECLCALLBACK(uint32_t) a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
550	/** Prototypes a native recompiler function for a threaded function. */
551	#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
552
553
554	DECLHIDDEN(uint32_t) iemNativeMakeLabel(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
555	uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
556	DECLHIDDEN(bool) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
557	IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0) RT_NOEXCEPT;
558	DECLHIDDEN(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
559	uint32_t cInstrReq) RT_NOEXCEPT;
560
561	DECLHIDDEN(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
562	bool fPreferVolatile = true) RT_NOEXCEPT;
563	DECLHIDDEN(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
564	bool fPreferVolatile = true) RT_NOEXCEPT;
565	DECLHIDDEN(uint8_t) iemNativeRegAllocTmpForGuest(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
566	IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
567	DECLHIDDEN(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar) RT_NOEXCEPT;
568	DECLHIDDEN(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs) RT_NOEXCEPT;
569	DECLHIDDEN(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
570	DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
571	DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
572	DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
573	DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
574	DECLHIDDEN(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off) RT_NOEXCEPT;
575
576	DECLHIDDEN(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off,
577	uint8_t idxInstr) RT_NOEXCEPT;
578
579
580	/**
581	* Ensures that there is sufficient space in the instruction output buffer.
582	*
583	* This will reallocate the buffer if needed and allowed.
584	*
585	* @returns Pointer to the instruction output buffer on success, NULL on
586	* failure.
587	* @param pReNative The native recompile state.
588	* @param off Current instruction offset. Works safely for UINT32_MAX
589	* as well.
590	* @param cInstrReq Number of instruction about to be added. It's okay to
591	* overestimate this a bit.
592	*/
593	DECL_FORCE_INLINE(PIEMNATIVEINSTR) iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
594	{
595	if (RT_LIKELY(off + (uint64_t)cInstrReq <= pReNative->cInstrBufAlloc))
596	return pReNative->pInstrBuf;
597	return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
598	}
599
600
601	/**
602	* Emit a simple marker instruction to more easily tell where something starts
603	* in the disassembly.
604	*/
605	DECLINLINE(uint32_t) iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
606	{
607	#ifdef RT_ARCH_AMD64
608	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
609	AssertReturn(pbCodeBuf, UINT32_MAX);
610	if (uInfo == 0)
611	{
612	/* nop */
613	pbCodeBuf[off++] = 0x90;
614	}
615	else
616	{
617	/* nop [disp32] */
618	pbCodeBuf[off++] = 0x0f;
619	pbCodeBuf[off++] = 0x1f;
620	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);
621	pbCodeBuf[off++] = RT_BYTE1(uInfo);
622	pbCodeBuf[off++] = RT_BYTE2(uInfo);
623	pbCodeBuf[off++] = RT_BYTE3(uInfo);
624	pbCodeBuf[off++] = RT_BYTE4(uInfo);
625	}
626	#elif RT_ARCH_ARM64
627	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
628	AssertReturn(pu32CodeBuf, UINT32_MAX);
629	/* nop */
630	pu32CodeBuf[off++] = 0xd503201f;
631
632	RT_NOREF(uInfo);
633	#else
634	# error "port me"
635	#endif
636	return off;
637	}
638
639
640	/**
641	* Emits setting a GPR to zero.
642	*/
643	DECLINLINE(uint32_t) iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
644	{
645	#ifdef RT_ARCH_AMD64
646	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
647	AssertReturn(pbCodeBuf, UINT32_MAX);
648	/* xor gpr32, gpr32 */
649	if (iGpr >= 8)
650	pbCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
651	pbCodeBuf[off++] = 0x33;
652	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
653
654	#elif RT_ARCH_ARM64
655	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
656	AssertReturn(pu32CodeBuf, UINT32_MAX);
657	/* mov gpr, #0x0 */
658	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| iGpr;
659
660	#else
661	# error "port me"
662	#endif
663	RT_NOREF(pReNative);
664	return off;
665	}
666
667
668	/**
669	* Emits loading a constant into a 64-bit GPR
670	*/
671	DECLINLINE(uint32_t) iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
672	{
673	if (!uImm64)
674	return iemNativeEmitGprZero(pReNative, off, iGpr);
675
676	#ifdef RT_ARCH_AMD64
677	if (uImm64 <= UINT32_MAX)
678	{
679	/* mov gpr, imm32 */
680	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
681	AssertReturn(pbCodeBuf, UINT32_MAX);
682	if (iGpr >= 8)
683	pbCodeBuf[off++] = X86_OP_REX_B;
684	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
685	pbCodeBuf[off++] = RT_BYTE1(uImm64);
686	pbCodeBuf[off++] = RT_BYTE2(uImm64);
687	pbCodeBuf[off++] = RT_BYTE3(uImm64);
688	pbCodeBuf[off++] = RT_BYTE4(uImm64);
689	}
690	else
691	{
692	/* mov gpr, imm64 */
693	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
694	AssertReturn(pbCodeBuf, UINT32_MAX);
695	if (iGpr < 8)
696	pbCodeBuf[off++] = X86_OP_REX_W;
697	else
698	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
699	pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
700	pbCodeBuf[off++] = RT_BYTE1(uImm64);
701	pbCodeBuf[off++] = RT_BYTE2(uImm64);
702	pbCodeBuf[off++] = RT_BYTE3(uImm64);
703	pbCodeBuf[off++] = RT_BYTE4(uImm64);
704	pbCodeBuf[off++] = RT_BYTE5(uImm64);
705	pbCodeBuf[off++] = RT_BYTE6(uImm64);
706	pbCodeBuf[off++] = RT_BYTE7(uImm64);
707	pbCodeBuf[off++] = RT_BYTE8(uImm64);
708	}
709
710	#elif RT_ARCH_ARM64
711	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
712	AssertReturn(pu32CodeBuf, UINT32_MAX);
713
714	/*
715	* We need to start this sequence with a 'mov grp, imm16, lsl #x' and
716	* supply remaining bits using 'movk grp, imm16, lsl #x'.
717	*
718	* The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
719	* while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
720	* is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
721	* after the first non-zero immediate component so we switch to movk for
722	* the remainder.
723	*/
724	uint32_t fMovK = 0;
725	/* mov gpr, imm16 */
726	uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
727	if (uImmPart)
728	{
729	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| uImmPart \| iGpr;
730	fMovK \|= RT_BIT_32(29);
731	}
732	/* mov[k] gpr, imm16, lsl #16 */
733	uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
734	if (uImmPart)
735	{
736	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(1) << 21) \| uImmPart \| iGpr;
737	fMovK \|= RT_BIT_32(29);
738	}
739	/* mov[k] gpr, imm16, lsl #32 */
740	uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
741	if (uImmPart)
742	{
743	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(2) << 21) \| uImmPart \| iGpr;
744	fMovK \|= RT_BIT_32(29);
745	}
746	/* mov[k] gpr, imm16, lsl #48 */
747	uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
748	if (uImmPart)
749	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| fMovK \| (UINT32_C(3) << 21) \| uImmPart \| iGpr;
750
751	/** @todo there is an inverted mask variant we might want to explore if it
752	* reduces the number of instructions... */
753	/** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
754	* clang 12.x does that, only to use the 'x' version for the
755	* addressing in the following ldr). */
756
757	#else
758	# error "port me"
759	#endif
760	return off;
761	}
762
763
764	/**
765	* Emits loading a constant into a 8-bit GPR
766	* @note The AMD64 version does NOT clear any bits in the 8..63 range,
767	* only the ARM64 version does that.
768	*/
769	DECLINLINE(uint32_t) iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)
770	{
771	#ifdef RT_ARCH_AMD64
772	/* mov gpr, imm8 */
773	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
774	AssertReturn(pbCodeBuf, UINT32_MAX);
775	if (iGpr >= 8)
776	pbCodeBuf[off++] = X86_OP_REX_B;
777	else if (iGpr >= 4)
778	pbCodeBuf[off++] = X86_OP_REX;
779	pbCodeBuf[off++] = 0xb0 + (iGpr & 7);
780	pbCodeBuf[off++] = RT_BYTE1(uImm8);
781
782	#elif RT_ARCH_ARM64
783	/* movz gpr, imm16, lsl #0 */
784	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
785	AssertReturn(pu32CodeBuf, UINT32_MAX);
786	pu32CodeBuf[off++] = UINT32_C(0xd2800000) \| (UINT32_C(0) << 21) \| ((uint32_t)uImm8 << 5) \| iGpr;
787
788	#else
789	# error "port me"
790	#endif
791	return off;
792	}
793
794
795	#ifdef RT_ARCH_AMD64
796	/**
797	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
798	*/
799	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
800	{
801	if (offVCpu < 128)
802	{
803	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
804	pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
805	}
806	else
807	{
808	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
809	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
810	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
811	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
812	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
813	}
814	return off;
815	}
816	#elif RT_ARCH_ARM64
817	/**
818	* Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
819	*/
820	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
821	uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
822	{
823	/*
824	* There are a couple of ldr variants that takes an immediate offset, so
825	* try use those if we can, otherwise we have to use the temporary register
826	* help with the addressing.
827	*/
828	if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
829	{
830	/* Use the unsigned variant of ldr Wt, [<Xn\|SP>, #off]. */
831	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
832	AssertReturn(pu32CodeBuf, UINT32_MAX);
833	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
834	}
835	else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
836	{
837	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
838	AssertReturn(pu32CodeBuf, UINT32_MAX);
839	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PCPUMCTX,
840	(offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
841	}
842	else
843	{
844	/* The offset is too large, so we must load it into a register and use
845	ldr Wt, [<Xn\|SP>, (<Wm>\|<Xm>)]. */
846	/** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
847	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
848
849	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
850	AssertReturn(pu32CodeBuf, UINT32_MAX);
851	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, IEMNATIVE_REG_FIXED_TMP);
852	}
853	return off;
854	}
855	#endif
856
857
858	/**
859	* Emits a 64-bit GPR load of a VCpu value.
860	*/
861	DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
862	{
863	#ifdef RT_ARCH_AMD64
864	/* mov reg64, mem64 */
865	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
866	AssertReturn(pbCodeBuf, UINT32_MAX);
867	if (iGpr < 8)
868	pbCodeBuf[off++] = X86_OP_REX_W;
869	else
870	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
871	pbCodeBuf[off++] = 0x8b;
872	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
873
874	#elif RT_ARCH_ARM64
875	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
876
877	#else
878	# error "port me"
879	#endif
880	return off;
881	}
882
883
884	/**
885	* Emits a 32-bit GPR load of a VCpu value.
886	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
887	*/
888	DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
889	{
890	#ifdef RT_ARCH_AMD64
891	/* mov reg32, mem32 */
892	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
893	AssertReturn(pbCodeBuf, UINT32_MAX);
894	if (iGpr >= 8)
895	pbCodeBuf[off++] = X86_OP_REX_R;
896	pbCodeBuf[off++] = 0x8b;
897	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
898
899	#elif RT_ARCH_ARM64
900	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
901
902	#else
903	# error "port me"
904	#endif
905	return off;
906	}
907
908
909	/**
910	* Emits a 16-bit GPR load of a VCpu value.
911	* @note Bits 16 thru 63 in the GPR will be zero after the operation.
912	*/
913	DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
914	{
915	#ifdef RT_ARCH_AMD64
916	/* movzx reg32, mem16 */
917	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
918	AssertReturn(pbCodeBuf, UINT32_MAX);
919	if (iGpr >= 8)
920	pbCodeBuf[off++] = X86_OP_REX_R;
921	pbCodeBuf[off++] = 0x0f;
922	pbCodeBuf[off++] = 0xb7;
923	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
924
925	#elif RT_ARCH_ARM64
926	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
927
928	#else
929	# error "port me"
930	#endif
931	return off;
932	}
933
934
935	/**
936	* Emits a 8-bit GPR load of a VCpu value.
937	* @note Bits 8 thru 63 in the GPR will be zero after the operation.
938	*/
939	DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
940	{
941	#ifdef RT_ARCH_AMD64
942	/* movzx reg32, mem8 */
943	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
944	AssertReturn(pbCodeBuf, UINT32_MAX);
945	if (iGpr >= 8)
946	pbCodeBuf[off++] = X86_OP_REX_R;
947	pbCodeBuf[off++] = 0x0f;
948	pbCodeBuf[off++] = 0xb6;
949	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
950
951	#elif RT_ARCH_ARM64
952	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
953
954	#else
955	# error "port me"
956	#endif
957	return off;
958	}
959
960
961	/**
962	* Emits a store of a GPR value to a 64-bit VCpu field.
963	*/
964	DECLINLINE(uint32_t) iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
965	{
966	#ifdef RT_ARCH_AMD64
967	/* mov mem64, reg64 */
968	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
969	AssertReturn(pbCodeBuf, UINT32_MAX);
970	if (iGpr < 8)
971	pbCodeBuf[off++] = X86_OP_REX_W;
972	else
973	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
974	pbCodeBuf[off++] = 0x89;
975	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
976
977	#elif RT_ARCH_ARM64
978	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
979
980	#else
981	# error "port me"
982	#endif
983	return off;
984	}
985
986
987	/**
988	* Emits a store of a GPR value to a 32-bit VCpu field.
989	*/
990	DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
991	{
992	#ifdef RT_ARCH_AMD64
993	/* mov mem32, reg32 */
994	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
995	AssertReturn(pbCodeBuf, UINT32_MAX);
996	if (iGpr >= 8)
997	pbCodeBuf[off++] = X86_OP_REX_R;
998	pbCodeBuf[off++] = 0x89;
999	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1000
1001	#elif RT_ARCH_ARM64
1002	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
1003
1004	#else
1005	# error "port me"
1006	#endif
1007	return off;
1008	}
1009
1010
1011	/**
1012	* Emits a store of a GPR value to a 16-bit VCpu field.
1013	*/
1014	DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1015	{
1016	#ifdef RT_ARCH_AMD64
1017	/* mov mem16, reg16 */
1018	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
1019	AssertReturn(pbCodeBuf, UINT32_MAX);
1020	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
1021	if (iGpr >= 8)
1022	pbCodeBuf[off++] = X86_OP_REX_R;
1023	pbCodeBuf[off++] = 0x89;
1024	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1025
1026	#elif RT_ARCH_ARM64
1027	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
1028
1029	#else
1030	# error "port me"
1031	#endif
1032	return off;
1033	}
1034
1035
1036	/**
1037	* Emits a store of a GPR value to a 8-bit VCpu field.
1038	*/
1039	DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
1040	{
1041	#ifdef RT_ARCH_AMD64
1042	/* mov mem8, reg8 */
1043	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1044	AssertReturn(pbCodeBuf, UINT32_MAX);
1045	if (iGpr >= 8)
1046	pbCodeBuf[off++] = X86_OP_REX_R;
1047	pbCodeBuf[off++] = 0x88;
1048	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
1049
1050	#elif RT_ARCH_ARM64
1051	off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
1052
1053	#else
1054	# error "port me"
1055	#endif
1056	return off;
1057	}
1058
1059
1060	/**
1061	* Emits a gprdst = gprsrc load.
1062	*/
1063	DECLINLINE(uint32_t) iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1064	{
1065	#ifdef RT_ARCH_AMD64
1066	/* mov gprdst, gprsrc */
1067	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1068	AssertReturn(pbCodeBuf, UINT32_MAX);
1069	if ((iGprDst \| iGprSrc) >= 8)
1070	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W \| X86_OP_REX_B
1071	: iGprSrc >= 8 ? X86_OP_REX_W \| X86_OP_REX_R \| X86_OP_REX_B
1072	: X86_OP_REX_W \| X86_OP_REX_R;
1073	else
1074	pbCodeBuf[off++] = X86_OP_REX_W;
1075	pbCodeBuf[off++] = 0x8b;
1076	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1077
1078	#elif RT_ARCH_ARM64
1079	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1080	AssertReturn(pu32CodeBuf, UINT32_MAX);
1081	/* mov dst, src; alias for: orr dst, xzr, src */
1082	pu32CodeBuf[off++] = UINT32_C(0xaa000000) \| ((uint32_t)iGprSrc << 16) \| ((uint32_t)ARMV8_A64_REG_XZR << 5) \| iGprDst;
1083
1084	#else
1085	# error "port me"
1086	#endif
1087	return off;
1088	}
1089
1090	#ifdef RT_ARCH_AMD64
1091	/**
1092	* Common bit of iemNativeEmitLoadGprByBp and friends.
1093	*/
1094	DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp)
1095	{
1096	if (offDisp < 128 && offDisp >= -128)
1097	{
1098	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1099	pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1100	}
1101	else
1102	{
1103	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1104	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1105	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1106	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1107	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1108	}
1109	return off;
1110	}
1111	#endif
1112
1113
1114	#ifdef RT_ARCH_AMD64
1115	/**
1116	* Emits a 64-bit GRP load instruction with an BP relative source address.
1117	*/
1118	DECLINLINE(uint32_t) iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1119	{
1120	/* mov gprdst, qword [rbp + offDisp] */
1121	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1122	AssertReturn(pbCodeBuf, UINT32_MAX);
1123	if (iGprDst < 8)
1124	pbCodeBuf[off++] = X86_OP_REX_W;
1125	else
1126	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1127	pbCodeBuf[off++] = 0x8b;
1128	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp);
1129	}
1130	#endif
1131
1132
1133	#ifdef RT_ARCH_AMD64
1134	/**
1135	* Emits a 32-bit GRP load instruction with an BP relative source address.
1136	*/
1137	DECLINLINE(uint32_t) iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1138	{
1139	/* mov gprdst, dword [rbp + offDisp] */
1140	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1141	AssertReturn(pbCodeBuf, UINT32_MAX);
1142	if (iGprDst >= 8)
1143	pbCodeBuf[off++] = X86_OP_REX_R;
1144	pbCodeBuf[off++] = 0x8b;
1145	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp);
1146	}
1147	#endif
1148
1149
1150	#ifdef RT_ARCH_AMD64
1151	/**
1152	* Emits a load effective address to a GRP with an BP relative source address.
1153	*/
1154	DECLINLINE(uint32_t) iemNativeEmitLeaGrpByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1155	{
1156	/* lea gprdst, [rbp + offDisp] */
1157	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1158	AssertReturn(pbCodeBuf, UINT32_MAX);
1159	if (iGprDst < 8)
1160	pbCodeBuf[off++] = X86_OP_REX_W;
1161	else
1162	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1163	pbCodeBuf[off++] = 0x8d;
1164	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp);
1165	}
1166	#endif
1167
1168
1169	/**
1170	* Emits a 64-bit GPR store with an BP relative destination address.
1171	*
1172	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1173	*/
1174	DECLINLINE(uint32_t) iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1175	{
1176	#ifdef RT_ARCH_AMD64
1177	/* mov qword [rbp + offDisp], gprdst */
1178	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1179	AssertReturn(pbCodeBuf, UINT32_MAX);
1180	if (iGprSrc < 8)
1181	pbCodeBuf[off++] = X86_OP_REX_W;
1182	else
1183	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_R;
1184	pbCodeBuf[off++] = 0x89;
1185	return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp);
1186
1187	#elif defined(RT_ARCH_ARM64)
1188	if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1189	{
1190	/* str w/ unsigned imm12 (scaled) */
1191	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1192	AssertReturn(pu32CodeBuf, UINT32_MAX);
1193	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1194	ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1195	}
1196	else if (offDisp >= -256 && offDisp <= 256)
1197	{
1198	/* stur w/ signed imm9 (unscaled) */
1199	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1200	AssertReturn(pu32CodeBuf, UINT32_MAX);
1201	pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1202	}
1203	else
1204	{
1205	/* Use temporary indexing register. */
1206	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1207	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1208	AssertReturn(pu32CodeBuf, UINT32_MAX);
1209	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1210	IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1211	}
1212	return off;
1213
1214	#else
1215	# error "Port me!"
1216	#endif
1217	}
1218
1219
1220	/**
1221	* Emits a 64-bit immediate store with an BP relative destination address.
1222	*
1223	* @note May trash IEMNATIVE_REG_FIXED_TMP0.
1224	*/
1225	DECLINLINE(uint32_t) iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1226	{
1227	#ifdef RT_ARCH_AMD64
1228	if ((int64_t)uImm64 == (int32_t)uImm64)
1229	{
1230	/* mov qword [rbp + offDisp], imm32 - sign extended */
1231	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1232	AssertReturn(pbCodeBuf, UINT32_MAX);
1233
1234	pbCodeBuf[off++] = X86_OP_REX_W;
1235	pbCodeBuf[off++] = 0xc7;
1236	if (offDisp < 128 && offDisp >= -128)
1237	{
1238	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1239	pbCodeBuf[off++] = (uint8_t)offDisp;
1240	}
1241	else
1242	{
1243	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1244	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1245	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1246	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1247	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1248	}
1249	pbCodeBuf[off++] = RT_BYTE1(uImm64);
1250	pbCodeBuf[off++] = RT_BYTE2(uImm64);
1251	pbCodeBuf[off++] = RT_BYTE3(uImm64);
1252	pbCodeBuf[off++] = RT_BYTE4(uImm64);
1253	return off;
1254	}
1255	#endif
1256
1257	/* Load tmp0, imm64; Store tmp to bp+disp. */
1258	off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1259	return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1260	}
1261
1262
1263	#ifdef RT_ARCH_AMD64
1264	/**
1265	* Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1266	*/
1267	DECLINLINE(uint32_t) iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1268	{
1269	/* sub gprdst, imm8/imm32 */
1270	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1271	AssertReturn(pbCodeBuf, UINT32_MAX);
1272	if (iGprDst < 8)
1273	pbCodeBuf[off++] = X86_OP_REX_W;
1274	else
1275	pbCodeBuf[off++] = X86_OP_REX_W \| X86_OP_REX_B;
1276	if (iSubtrahend < 128 && iSubtrahend >= -128)
1277	{
1278	pbCodeBuf[off++] = 0x83;
1279	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1280	pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1281	}
1282	else
1283	{
1284	pbCodeBuf[off++] = 0x81;
1285	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1286	pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1287	pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1288	pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1289	pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1290	}
1291	return off;
1292	}
1293	#endif
1294
1295
1296	/**
1297	* Emits adding a 64-bit GPR to another, storing the result in the frist.
1298	* @note The AMD64 version sets flags.
1299	*/
1300	DECLINLINE(uint32_t ) iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
1301	{
1302	#if defined(RT_ARCH_AMD64)
1303	/* add Gv,Ev */
1304	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1305	AssertReturn(pbCodeBuf, UINT32_MAX);
1306	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1307	\| (iGprAddend < 8 ? 0 : X86_OP_REX_B);
1308	pbCodeBuf[off++] = 0x04;
1309	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
1310
1311	#elif defined(RT_ARCH_ARM64)
1312	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1313	AssertReturn(pu32CodeBuf, UINT32_MAX);
1314	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iGprAddend);
1315
1316	#else
1317	# error "Port me"
1318	#endif
1319	return off;
1320	}
1321
1322
1323	/**
1324	* Emits a 64-bit GPR additions with a 8-bit signed immediate.
1325	*/
1326	DECLINLINE(uint32_t ) iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1327	{
1328	#if defined(RT_ARCH_AMD64)
1329	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1330	AssertReturn(pbCodeBuf, UINT32_MAX);
1331	/* add or inc */
1332	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1333	if (iImm8 != 1)
1334	{
1335	pbCodeBuf[off++] = 0x83;
1336	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1337	pbCodeBuf[off++] = (uint8_t)iImm8;
1338	}
1339	else
1340	{
1341	pbCodeBuf[off++] = 0xff;
1342	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1343	}
1344
1345	#elif defined(RT_ARCH_ARM64)
1346	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1347	AssertReturn(pu32CodeBuf, UINT32_MAX);
1348	if (iImm8 >= 0)
1349	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8);
1350	else
1351	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8);
1352
1353	#else
1354	# error "Port me"
1355	#endif
1356	return off;
1357	}
1358
1359
1360	/**
1361	* Emits a 32-bit GPR additions with a 8-bit signed immediate.
1362	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1363	*/
1364	DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1365	{
1366	#if defined(RT_ARCH_AMD64)
1367	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1368	AssertReturn(pbCodeBuf, UINT32_MAX);
1369	/* add or inc */
1370	if (iGprDst >= 8)
1371	pbCodeBuf[off++] = X86_OP_REX_B;
1372	if (iImm8 != 1)
1373	{
1374	pbCodeBuf[off++] = 0x83;
1375	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1376	pbCodeBuf[off++] = (uint8_t)iImm8;
1377	}
1378	else
1379	{
1380	pbCodeBuf[off++] = 0xff;
1381	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1382	}
1383
1384	#elif defined(RT_ARCH_ARM64)
1385	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1386	AssertReturn(pu32CodeBuf, UINT32_MAX);
1387	if (iImm8 >= 0)
1388	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint8_t)iImm8, false /f64Bit/);
1389	else
1390	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint8_t)-iImm8, false /f64Bit/);
1391
1392	#else
1393	# error "Port me"
1394	#endif
1395	return off;
1396	}
1397
1398
1399	/**
1400	* Emits a 64-bit GPR additions with a 64-bit signed addend.
1401	*/
1402	DECLINLINE(uint32_t ) iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)
1403	{
1404	#if defined(RT_ARCH_AMD64)
1405	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1406	return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);
1407
1408	if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)
1409	{
1410	/* add grp, imm32 */
1411	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1412	AssertReturn(pbCodeBuf, UINT32_MAX);
1413	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1414	pbCodeBuf[off++] = 0x81;
1415	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1416	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1417	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1418	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1419	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1420	}
1421	else
1422	{
1423	/* Best to use a temporary register to deal with this in the simplest way: */
1424	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1425	AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1426
1427	/* add dst, tmpreg */
1428	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1429	AssertReturn(pbCodeBuf, UINT32_MAX);
1430	pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R)
1431	\| (iTmpReg < 8 ? 0 : X86_OP_REX_B);
1432	pbCodeBuf[off++] = 0x03;
1433	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);
1434
1435	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1436	}
1437
1438	#elif defined(RT_ARCH_ARM64)
1439	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1440	{
1441	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1442	AssertReturn(pu32CodeBuf, UINT32_MAX);
1443	if (iAddend >= 0)
1444	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend);
1445	else
1446	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend);
1447	}
1448	else
1449	{
1450	/* Use temporary register for the immediate. */
1451	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);
1452	AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1453
1454	/* add gprdst, gprdst, tmpreg */
1455	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1456	AssertReturn(pu32CodeBuf, UINT32_MAX);
1457	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg);
1458
1459	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1460	}
1461
1462	#else
1463	# error "Port me"
1464	#endif
1465	return off;
1466	}
1467
1468
1469	/**
1470	* Emits a 32-bit GPR additions with a 32-bit signed immediate.
1471	* @note Bits 32 thru 63 in the GPR will be zero after the operation.
1472	*/
1473	DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
1474	{
1475	#if defined(RT_ARCH_AMD64)
1476	if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
1477	return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
1478
1479	/* add grp, imm32 */
1480	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1481	AssertReturn(pbCodeBuf, UINT32_MAX);
1482	if (iGprDst >= 8)
1483	pbCodeBuf[off++] = X86_OP_REX_B;
1484	pbCodeBuf[off++] = 0x81;
1485	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1486	pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
1487	pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
1488	pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
1489	pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
1490
1491	#elif defined(RT_ARCH_ARM64)
1492	if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))
1493	{
1494	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1495	AssertReturn(pu32CodeBuf, UINT32_MAX);
1496	if (iAddend >= 0)
1497	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, iGprDst, iGprDst, (uint32_t)iAddend, false /f64Bit/);
1498	else
1499	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /fSub/, iGprDst, iGprDst, (uint32_t)-iAddend, false /f64Bit/);
1500	}
1501	else
1502	{
1503	/* Use temporary register for the immediate. */
1504	uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);
1505	AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->aHstRegs), UINT32_MAX);
1506
1507	/* add gprdst, gprdst, tmpreg */
1508	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1509	AssertReturn(pu32CodeBuf, UINT32_MAX);
1510	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /fSub/, iGprDst, iGprDst, iTmpReg, false /f64Bit/);
1511
1512	iemNativeRegFreeTmpImm(pReNative, iTmpReg);
1513	}
1514
1515	#else
1516	# error "Port me"
1517	#endif
1518	return off;
1519	}
1520
1521
1522	/**
1523	* Emits code for clearing bits 16 thru 63 in the GPR.
1524	*/
1525	DECLINLINE(uint32_t ) iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
1526	{
1527	#if defined(RT_ARCH_AMD64)
1528	/* movzx reg32, reg16 */
1529	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1530	AssertReturn(pbCodeBuf, UINT32_MAX);
1531	if (iGprDst >= 8)
1532	pbCodeBuf[off++] = X86_OP_REX_B \| X86_OP_REX_R;
1533	pbCodeBuf[off++] = 0x0f;
1534	pbCodeBuf[off++] = 0xb7;
1535	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);
1536
1537	#elif defined(RT_ARCH_ARM64)
1538	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1539	AssertReturn(pu32CodeBuf, UINT32_MAX);
1540	# if 1
1541	pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);
1542	# else
1543	///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */
1544	//pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);
1545	# endif
1546	#else
1547	# error "Port me"
1548	#endif
1549	return off;
1550	}
1551
1552
1553	/**
1554	* Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
1555	*/
1556	DECLINLINE(uint32_t ) iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
1557	{
1558	#if defined(RT_ARCH_AMD64)
1559	/* shr dst, cShift */
1560	uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1561	AssertReturn(pbCodeBuf, UINT32_MAX);
1562	pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_B;
1563	pbCodeBuf[off++] = 0xc0;
1564	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1565	pbCodeBuf[off++] = cShift;
1566	Assert(cShift > 0 && cShift < 64);
1567
1568	#elif defined(RT_ARCH_ARM64)
1569	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1570	AssertReturn(pu32CodeBuf, UINT32_MAX);
1571	pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
1572	#else
1573	# error "Port me"
1574	#endif
1575	return off;
1576	}
1577
1578
1579	#ifdef RT_ARCH_ARM64
1580	/**
1581	* Emits an ARM64 compare instruction.
1582	*/
1583	DECLINLINE(uint32_t) iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,
1584	bool f64Bit = true, uint32_t cShift = 0,
1585	ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)
1586	{
1587	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1588	AssertReturn(pu32CodeBuf, UINT32_MAX);
1589	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR /iRegResult/, iGprLeft, iGprRight,
1590	f64Bit, true /fSetFlags/, cShift, enmShift);
1591	return off;
1592	}
1593	#endif
1594
1595
1596	/**
1597	* Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
1598	* with conditional instruction.
1599	*/
1600	DECLINLINE(uint32_t) iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
1601	{
1602	#ifdef RT_ARCH_AMD64
1603	/* cmp Gv, Ev */
1604	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1605	AssertReturn(pbCodeBuf, UINT32_MAX);
1606	pbCodeBuf[off++] = X86_OP_REX_W \| (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
1607	pbCodeBuf[off++] = 0x3b;
1608	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1609
1610	#elif defined(RT_ARCH_ARM64)
1611	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
1612
1613	#else
1614	# error "Port me!"
1615	#endif
1616	return off;
1617	}
1618
1619
1620	/**
1621	* Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
1622	* with conditional instruction.
1623	*/
1624	DECLINLINE(uint32_t) iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1625	uint8_t iGprLeft, uint8_t iGprRight)
1626	{
1627	#ifdef RT_ARCH_AMD64
1628	/* cmp Gv, Ev */
1629	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1630	AssertReturn(pbCodeBuf, UINT32_MAX);
1631	if (iGprLeft >= 8 \|\| iGprRight >= 8)
1632	pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) \| (iGprRight >= 8 ? X86_OP_REX_B : 0);
1633	pbCodeBuf[off++] = 0x3b;
1634	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
1635
1636	#elif defined(RT_ARCH_ARM64)
1637	off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /f64Bit/);
1638
1639	#else
1640	# error "Port me!"
1641	#endif
1642	return off;
1643	}
1644
1645
1646	/**
1647	* Emits a JMP rel32 / B imm19 to the given label (ASSUMED requiring fixup).
1648	*/
1649	DECLINLINE(uint32_t) iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
1650	{
1651	#ifdef RT_ARCH_AMD64
1652	/* jnz rel32 */
1653	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
1654	AssertReturn(pbCodeBuf, UINT32_MAX);
1655	pbCodeBuf[off++] = 0xe9;
1656	AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
1657	pbCodeBuf[off++] = 0xfe;
1658	pbCodeBuf[off++] = 0xff;
1659	pbCodeBuf[off++] = 0xff;
1660	pbCodeBuf[off++] = 0xff;
1661
1662	#elif defined(RT_ARCH_ARM64)
1663	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1664	AssertReturn(pu32CodeBuf, UINT32_MAX);
1665	AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
1666	pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
1667
1668	#else
1669	# error "Port me!"
1670	#endif
1671	return off;
1672	}
1673
1674
1675	/**
1676	* Emits a JMP rel32 / B imm19 to a new undefined label.
1677	*/
1678	DECLINLINE(uint32_t) iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1679	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1680	{
1681	uint32_t const idxLabel = iemNativeMakeLabel(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
1682	AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
1683	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
1684	}
1685
1686	/** Condition type. */
1687	#ifdef RT_ARCH_AMD64
1688	typedef uint8_t IEMNATIVEINSTRCOND;
1689	#elif defined(RT_ARCH_ARM64)
1690	typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;
1691	#else
1692	# error "Port me!"
1693	#endif
1694
1695
1696	/**
1697	* Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
1698	*/
1699	DECLINLINE(uint32_t) iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1700	uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
1701	{
1702	#ifdef RT_ARCH_AMD64
1703	/* jcc rel32 */
1704	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
1705	AssertReturn(pbCodeBuf, UINT32_MAX);
1706	pbCodeBuf[off++] = 0x0f;
1707	pbCodeBuf[off++] = enmCond \| 0x80;
1708	AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4), UINT32_MAX);
1709	pbCodeBuf[off++] = 0x00;
1710	pbCodeBuf[off++] = 0x00;
1711	pbCodeBuf[off++] = 0x00;
1712	pbCodeBuf[off++] = 0x00;
1713
1714	#elif defined(RT_ARCH_ARM64)
1715	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1716	AssertReturn(pu32CodeBuf, UINT32_MAX);
1717	AssertReturn(iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5), UINT32_MAX);
1718	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
1719
1720	#else
1721	# error "Port me!"
1722	#endif
1723	return off;
1724	}
1725
1726
1727	/**
1728	* Emits a Jcc rel32 / B.cc imm19 to a new label.
1729	*/
1730	DECLINLINE(uint32_t) iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1731	IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)
1732	{
1733	uint32_t const idxLabel = iemNativeMakeLabel(pReNative, enmLabelType, UINT32_MAX /offWhere/, uData);
1734	AssertReturn(idxLabel != UINT32_MAX, UINT32_MAX);
1735	return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);
1736	}
1737
1738
1739	/**
1740	* Emits a JZ/JE rel32 / B.EQ imm19 to a new label.
1741	*/
1742	DECLINLINE(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1743	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1744	{
1745	#ifdef RT_ARCH_AMD64
1746	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, 0x4);
1747	#elif defined(RT_ARCH_ARM64)
1748	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);
1749	#else
1750	# error "Port me!"
1751	#endif
1752	}
1753
1754
1755	/**
1756	* Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.
1757	*/
1758	DECLINLINE(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1759	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1760	{
1761	#ifdef RT_ARCH_AMD64
1762	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, 0x5);
1763	#elif defined(RT_ARCH_ARM64)
1764	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);
1765	#else
1766	# error "Port me!"
1767	#endif
1768	}
1769
1770
1771	/**
1772	* Emits a JBE/JNA rel32 / B.LS imm19 to a new label.
1773	*/
1774	DECLINLINE(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1775	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1776	{
1777	#ifdef RT_ARCH_AMD64
1778	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, 0x6);
1779	#elif defined(RT_ARCH_ARM64)
1780	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);
1781	#else
1782	# error "Port me!"
1783	#endif
1784	}
1785
1786
1787	/**
1788	* Emits a JA/JNBE rel32 / B.HI imm19 to a new label.
1789	*/
1790	DECLINLINE(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1791	IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)
1792	{
1793	#ifdef RT_ARCH_AMD64
1794	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, 0x7);
1795	#elif defined(RT_ARCH_ARM64)
1796	return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);
1797	#else
1798	# error "Port me!"
1799	#endif
1800	}
1801
1802
1803	/**
1804	* Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
1805	* How @a offJmp is applied is are target specific.
1806	*/
1807	DECLINLINE(uint32_t) iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1808	int32_t offTarget, IEMNATIVEINSTRCOND enmCond)
1809	{
1810	#ifdef RT_ARCH_AMD64
1811	/* jcc rel32 */
1812	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
1813	AssertReturn(pbCodeBuf, UINT32_MAX);
1814	if (offTarget < 128 && offTarget >= -128)
1815	{
1816	pbCodeBuf[off++] = enmCond \| 0x70;
1817	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
1818	}
1819	else
1820	{
1821	pbCodeBuf[off++] = 0x0f;
1822	pbCodeBuf[off++] = enmCond \| 0x80;
1823	pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);
1824	pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);
1825	pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);
1826	pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);
1827	}
1828
1829	#elif defined(RT_ARCH_ARM64)
1830	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1831	AssertReturn(pu32CodeBuf, UINT32_MAX);
1832	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);
1833
1834	#else
1835	# error "Port me!"
1836	#endif
1837	return off;
1838	}
1839
1840
1841	/**
1842	* Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.
1843	* How @a offJmp is applied is are target specific.
1844	*/
1845	DECLINLINE(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
1846	{
1847	#ifdef RT_ARCH_AMD64
1848	return iemNativeEmitJccToFixed(pReNative, off, offTarget, 0x4);
1849	#elif defined(RT_ARCH_ARM64)
1850	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);
1851	#else
1852	# error "Port me!"
1853	#endif
1854	}
1855
1856
1857	/**
1858	* Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.
1859	* How @a offJmp is applied is are target specific.
1860	*/
1861	DECLINLINE(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
1862	{
1863	#ifdef RT_ARCH_AMD64
1864	return iemNativeEmitJccToFixed(pReNative, off, offTarget, 0x5);
1865	#elif defined(RT_ARCH_ARM64)
1866	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);
1867	#else
1868	# error "Port me!"
1869	#endif
1870	}
1871
1872
1873	/**
1874	* Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.
1875	* How @a offJmp is applied is are target specific.
1876	*/
1877	DECLINLINE(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
1878	{
1879	#ifdef RT_ARCH_AMD64
1880	return iemNativeEmitJccToFixed(pReNative, off, offTarget, 0x6);
1881	#elif defined(RT_ARCH_ARM64)
1882	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);
1883	#else
1884	# error "Port me!"
1885	#endif
1886	}
1887
1888
1889	/**
1890	* Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.
1891	* How @a offJmp is applied is are target specific.
1892	*/
1893	DECLINLINE(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)
1894	{
1895	#ifdef RT_ARCH_AMD64
1896	return iemNativeEmitJccToFixed(pReNative, off, offTarget, 0x7);
1897	#elif defined(RT_ARCH_ARM64)
1898	return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);
1899	#else
1900	# error "Port me!"
1901	#endif
1902	}
1903
1904
1905	/**
1906	* Fixes up a conditional jump to a fixed label.
1907	* @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...
1908	*/
1909	DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)
1910	{
1911	# if defined(RT_ARCH_AMD64)
1912	uint8_t * const pbCodeBuf = pReNative->pInstrBuf;
1913	if (pbCodeBuf[offFixup] != 0x0f)
1914	{
1915	Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);
1916	pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));
1917	Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));
1918	}
1919	else
1920	{
1921	Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);
1922	uint32_t const offRel32 = offTarget - (offFixup + 6);
1923	pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);
1924	pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);
1925	pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);
1926	pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);
1927	}
1928
1929	# elif defined(RT_ARCH_ARM64)
1930	uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
1931	Assert(RT_ABS((int32_t)(offTarget - offFixup)) < RT_BIT_32(18)); /* off by one for negative jumps, but not relevant here */
1932	pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & ~((RT_BIT_32(19) - 1U) << 5))
1933	\| (((offTarget - offFixup) & (RT_BIT_32(19) - 1U)) << 5);
1934
1935	# endif
1936	}
1937
1938
1939
1940	/** @} */
1941
1942	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
1943

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 101521

Download in other formats: