IEMAllN8veRecompBltIn.cpp@ 103583

Last change on this file since 103583 was 103318, checked in by vboxsync, 13 months ago
VMM/IEM: Liveness analysis, part 10: Debugging, asserting liveness state sanity, major fixes, new storage format. bugref:10372
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 95.7 KB

Line
1	/* $Id: IEMAllN8veRecompBltIn.cpp 103318 2024-02-12 16:24:58Z vboxsync $ */
2	/** @file
3	* IEM - Native Recompiler, Emitters for Built-In Threaded Functions.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
33	//#define IEM_WITH_OPAQUE_DECODER_STATE - need offCurInstrStart access for iemNativeHlpMemCodeNewPageTlbMiss and friends.
34	#define VMCPU_INCL_CPUM_GST_CTX
35	#define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
36	#include <VBox/vmm/iem.h>
37	#include <VBox/vmm/cpum.h>
38	#include <VBox/vmm/dbgf.h>
39	#include "IEMInternal.h"
40	#include <VBox/vmm/vmcc.h>
41	#include <VBox/log.h>
42	#include <VBox/err.h>
43	#include <VBox/param.h>
44	#include <iprt/assert.h>
45	#include <iprt/string.h>
46	#if defined(RT_ARCH_AMD64)
47	# include <iprt/x86.h>
48	#elif defined(RT_ARCH_ARM64)
49	# include <iprt/armv8.h>
50	#endif
51
52
53	#include "IEMInline.h"
54	#include "IEMThreadedFunctions.h"
55	#include "IEMN8veRecompiler.h"
56	#include "IEMN8veRecompilerEmit.h"
57	#include "IEMN8veRecompilerTlbLookup.h"
58
59
60
61	/*********************************************************************************************************************************
62	* TB Helper Functions *
63	*********************************************************************************************************************************/
64	#ifdef RT_ARCH_AMD64
65	DECLASM(void) iemNativeHlpAsmSafeWrapLogCpuState(void);
66	#endif
67
68
69	/**
70	* Used by TB code to deal with a TLB miss for a new page.
71	*/
72	IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemCodeNewPageTlbMiss,(PVMCPUCC pVCpu))
73	{
74	STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage);
75	pVCpu->iem.s.pbInstrBuf = NULL;
76	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE;
77	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
78	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
79	if (pVCpu->iem.s.pbInstrBuf)
80	{ /* likely */ }
81	else
82	{
83	IEM_DO_LONGJMP(pVCpu, VINF_IEM_REEXEC_BREAK);
84	}
85	}
86
87
88	/**
89	* Used by TB code to deal with a TLB miss for a new page.
90	*/
91	IEM_DECL_NATIVE_HLP_DEF(RTGCPHYS, iemNativeHlpMemCodeNewPageTlbMissWithOff,(PVMCPUCC pVCpu, uint8_t offInstr))
92	{
93	STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset);
94	pVCpu->iem.s.pbInstrBuf = NULL;
95	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE - offInstr;
96	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
97	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
98	return pVCpu->iem.s.pbInstrBuf ? pVCpu->iem.s.GCPhysInstrBuf : NIL_RTGCPHYS;
99	}
100
101
102	/*********************************************************************************************************************************
103	* Builtin functions *
104	*********************************************************************************************************************************/
105
106	/**
107	* Built-in function that does nothing.
108	*
109	* Whether this is called or not can be controlled by the entry in the
110	* IEMThreadedGenerator.katBltIns table. This can be useful to determine
111	* whether why behaviour changes when enabling the LogCpuState builtins. I.e.
112	* whether it's the reduced call count in the TBs or the threaded calls flushing
113	* register state.
114	*/
115	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Nop)
116	{
117	RT_NOREF(pReNative, pCallEntry);
118	return off;
119	}
120
121	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Nop)
122	{
123	pOutgoing = pIncoming;
124	RT_NOREF(pCallEntry);
125	}
126
127
128	/**
129	* Emits for for LogCpuState.
130	*
131	* This shouldn't have any relevant impact on the recompiler state.
132	*/
133	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_LogCpuState)
134	{
135	#ifdef RT_ARCH_AMD64
136	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 20);
137	/* push rax */
138	pbCodeBuf[off++] = 0x50 + X86_GREG_xAX;
139	/* push imm32 */
140	pbCodeBuf[off++] = 0x68;
141	pbCodeBuf[off++] = RT_BYTE1(pCallEntry->auParams[0]);
142	pbCodeBuf[off++] = RT_BYTE2(pCallEntry->auParams[0]);
143	pbCodeBuf[off++] = RT_BYTE3(pCallEntry->auParams[0]);
144	pbCodeBuf[off++] = RT_BYTE4(pCallEntry->auParams[0]);
145	/* mov rax, iemNativeHlpAsmSafeWrapLogCpuState */
146	pbCodeBuf[off++] = X86_OP_REX_W;
147	pbCodeBuf[off++] = 0xb8 + X86_GREG_xAX;
148	(uint64_t )&pbCodeBuf[off] = (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState;
149	off += sizeof(uint64_t);
150	/* call rax */
151	pbCodeBuf[off++] = 0xff;
152	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
153	/* pop rax */
154	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
155	/* pop rax */
156	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
157	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
158
159	#else
160	/** @todo Implement this */
161	AssertFailed();
162	RT_NOREF(pReNative, pCallEntry);
163	#endif
164	return off;
165	}
166
167	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_LogCpuState)
168	{
169	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
170	RT_NOREF(pCallEntry);
171	}
172
173
174	/**
175	* Built-in function that calls a C-implemention function taking zero arguments.
176	*/
177	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_DeferToCImpl0)
178	{
179	PFNIEMCIMPL0 const pfnCImpl = (PFNIEMCIMPL0)(uintptr_t)pCallEntry->auParams[0];
180	uint8_t const cbInstr = (uint8_t)pCallEntry->auParams[1];
181	uint64_t const fGstShwFlush = pCallEntry->auParams[2];
182	return iemNativeEmitCImplCall(pReNative, off, pCallEntry->idxInstr, fGstShwFlush, (uintptr_t)pfnCImpl, cbInstr, 0, 0, 0, 0);
183	}
184
185	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_DeferToCImpl0)
186	{
187	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
188	RT_NOREF(pCallEntry);
189	}
190
191
192	/**
193	* Built-in function that checks for pending interrupts that can be delivered or
194	* forced action flags.
195	*
196	* This triggers after the completion of an instruction, so EIP is already at
197	* the next instruction. If an IRQ or important FF is pending, this will return
198	* a non-zero status that stops TB execution.
199	*/
200	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
201	{
202	RT_NOREF(pCallEntry);
203
204	/* It's too convenient to use iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet below
205	and I'm too lazy to create a 'Fixed' version of that one. */
206	uint32_t const idxLabelVmCheck = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckIrq,
207	UINT32_MAX, pReNative->uCheckIrqSeqNo++);
208
209	uint32_t const idxLabelReturnBreak = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ReturnBreak);
210
211	/* Again, we need to load the extended EFLAGS before we actually need them
212	in case we jump. We couldn't use iemNativeRegAllocTmpForGuestReg if we
213	loaded them inside the check, as the shadow state would not be correct
214	when the code branches before the load. Ditto PC. */
215	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
216	kIemNativeGstRegUse_ReadOnly);
217
218	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ReadOnly);
219
220	uint8_t idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
221
222	/*
223	* Start by checking the local forced actions of the EMT we're on for IRQs
224	* and other FFs that needs servicing.
225	*/
226	/** @todo this isn't even close to the NMI and interrupt conditions in EM! */
227	/* Load FFs in to idxTmpReg and AND with all relevant flags. */
228	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
229	off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
230	VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
231	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
232	\| VMCPU_FF_TLB_FLUSH
233	\| VMCPU_FF_UNHALT ),
234	true /fSetFlags/);
235	/* If we end up with ZERO in idxTmpReg there is nothing to do.*/
236	uint32_t const offFixupJumpToVmCheck1 = off;
237	off = iemNativeEmitJzToFixed(pReNative, off, off /* ASSUME jz rel8 suffices */);
238
239	/* Some relevant FFs are set, but if's only APIC or/and PIC being set,
240	these may be supressed by EFLAGS.IF or CPUMIsInInterruptShadow. */
241	off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
242	~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC), true /fSetFlags/);
243	/* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
244	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreak);
245
246	/* So, it's only interrupt releated FFs and we need to see if IRQs are being
247	suppressed by the CPU or not. */
248	off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
249	off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
250	idxLabelReturnBreak);
251
252	/* We've got shadow flags set, so we must check that the PC they are valid
253	for matches our current PC value. */
254	/** @todo AMD64 can do this more efficiently w/o loading uRipInhibitInt into
255	* a register. */
256	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
257	off = iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, idxTmpReg, idxPcReg, idxLabelReturnBreak);
258
259	/*
260	* Now check the force flags of the VM.
261	*/
262	iemNativeLabelDefine(pReNative, idxLabelVmCheck, off);
263	iemNativeFixupFixedJump(pReNative, offFixupJumpToVmCheck1, off);
264	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, CTX_SUFF(pVM))); /* idxTmpReg = pVM */
265	off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
266	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /fSetFlags/);
267	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreak);
268
269	/** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
270
271	/*
272	* We're good, no IRQs or FFs pending.
273	*/
274	iemNativeRegFreeTmp(pReNative, idxTmpReg);
275	iemNativeRegFreeTmp(pReNative, idxEflReg);
276	iemNativeRegFreeTmp(pReNative, idxPcReg);
277
278	return off;
279	}
280
281	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckIrq)
282	{
283	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
284	IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
285	RT_NOREF(pCallEntry);
286	}
287
288
289	/**
290	* Built-in function checks if IEMCPU::fExec has the expected value.
291	*/
292	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckMode)
293	{
294	uint32_t const fExpectedExec = (uint32_t)pCallEntry->auParams[0];
295	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
296
297	off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
298	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
299	off = iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
300	kIemNativeLabelType_ReturnBreak);
301	iemNativeRegFreeTmp(pReNative, idxTmpReg);
302
303	/* Maintain the recompiler fExec state. */
304	pReNative->fExec = fExpectedExec & IEMTB_F_IEM_F_MASK;
305	return off;
306	}
307
308	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckMode)
309	{
310	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
311	RT_NOREF(pCallEntry);
312	}
313
314
315	/**
316	* Sets idxTbCurInstr in preparation of raising an exception or aborting the TB.
317	*/
318	/** @todo Optimize this, so we don't set the same value more than once. Just
319	* needs some tracking. */
320	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
321	# define BODY_SET_CUR_INSTR() \
322	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, pCallEntry->idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr))
323	#else
324	# define BODY_SET_CUR_INSTR() ((void)0)
325	#endif
326
327	/**
328	* Flushes pending writes in preparation of raising an exception or aborting the TB.
329	*/
330	#define BODY_FLUSH_PENDING_WRITES() \
331	off = iemNativeRegFlushPendingWrites(pReNative, off);
332
333
334	/**
335	* Macro that emits the 16/32-bit CS.LIM check.
336	*/
337	#define BODY_CHECK_CS_LIM(a_cbInstr) \
338	off = iemNativeEmitBltInCheckCsLim(pReNative, off, (a_cbInstr))
339
340	#define LIVENESS_CHECK_CS_LIM(a_pOutgoing) \
341	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS)
342
343	DECL_FORCE_INLINE(uint32_t)
344	iemNativeEmitBltInCheckCsLim(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
345	{
346	Assert(cbInstr > 0);
347	Assert(cbInstr < 16);
348	#ifdef VBOX_STRICT
349	off = iemNativeEmitMarker(pReNative, off, 0x80000001);
350	#endif
351
352	/*
353	* We need CS.LIM and RIP here. When cbInstr is larger than 1, we also need
354	* a temporary register for calculating the last address of the instruction.
355	*
356	* The calculation and comparisons are 32-bit. We ASSUME that the incoming
357	* RIP isn't totally invalid, i.e. that any jump/call/ret/iret instruction
358	* that last updated EIP here checked it already, and that we're therefore
359	* safe in the 32-bit wrap-around scenario to only check that the last byte
360	* is within CS.LIM. In the case of instruction-by-instruction advancing
361	* up to a EIP wrap-around, we know that CS.LIM is 4G-1 because the limit
362	* must be using 4KB granularity and the previous instruction was fine.
363	*/
364	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
365	kIemNativeGstRegUse_ReadOnly);
366	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
367	kIemNativeGstRegUse_ReadOnly);
368	#ifdef RT_ARCH_AMD64
369	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
370	#elif defined(RT_ARCH_ARM64)
371	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
372	#else
373	# error "Port me"
374	#endif
375
376	if (cbInstr != 1)
377	{
378	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
379
380	/*
381	* 1. idxRegTmp = idxRegPc + cbInstr;
382	* 2. if idxRegTmp > idxRegCsLim then raise #GP(0).
383	*/
384	#ifdef RT_ARCH_AMD64
385	/* 1. lea tmp32, [Pc + cbInstr - 1] */
386	if (idxRegTmp >= 8 \|\| idxRegPc >= 8)
387	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegPc < 8 ? 0 : X86_OP_REX_B);
388	pbCodeBuf[off++] = 0x8d;
389	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, idxRegTmp & 7, idxRegPc & 7);
390	if ((idxRegPc & 7) == X86_GREG_xSP)
391	pbCodeBuf[off++] = X86_SIB_MAKE(idxRegPc & 7, 4 /no index/, 0);
392	pbCodeBuf[off++] = cbInstr - 1;
393
394	/* 2. cmp tmp32(r), CsLim(r/m). */
395	if (idxRegTmp >= 8 \|\| idxRegCsLim >= 8)
396	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
397	pbCodeBuf[off++] = 0x3b;
398	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegTmp & 7, idxRegCsLim & 7);
399
400	#elif defined(RT_ARCH_ARM64)
401	/* 1. add tmp32, Pc, #cbInstr-1 */
402	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxRegTmp, idxRegPc, cbInstr - 1, false /f64Bit/);
403	/* 2. cmp tmp32, CsLim */
404	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegTmp, idxRegCsLim,
405	false /f64Bit/, true /fSetFlags/);
406
407	#endif
408	iemNativeRegFreeTmp(pReNative, idxRegTmp);
409	}
410	else
411	{
412	/*
413	* Here we can skip step 1 and compare PC and CS.LIM directly.
414	*/
415	#ifdef RT_ARCH_AMD64
416	/* 2. cmp eip(r), CsLim(r/m). */
417	if (idxRegPc >= 8 \|\| idxRegCsLim >= 8)
418	pbCodeBuf[off++] = (idxRegPc < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
419	pbCodeBuf[off++] = 0x3b;
420	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegPc & 7, idxRegCsLim & 7);
421
422	#elif defined(RT_ARCH_ARM64)
423	/* 2. cmp Pc, CsLim */
424	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegPc, idxRegCsLim,
425	false /f64Bit/, true /fSetFlags/);
426
427	#endif
428	}
429
430	/* 3. Jump if greater. */
431	off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_RaiseGp0);
432
433	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
434	iemNativeRegFreeTmp(pReNative, idxRegPc);
435	return off;
436	}
437
438
439	/**
440	* Macro that considers whether we need CS.LIM checking after a branch or
441	* crossing over to a new page.
442	*/
443	#define BODY_CONSIDER_CS_LIM_CHECKING(a_pTb, a_cbInstr) \
444	RT_NOREF(a_cbInstr); \
445	off = iemNativeEmitBltInConsiderLimChecking(pReNative, off)
446
447	#define LIVENESS_CONSIDER_CS_LIM_CHECKING(a_pOutgoing) \
448	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS); \
449	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS)
450
451	DECL_FORCE_INLINE(uint32_t)
452	iemNativeEmitBltInConsiderLimChecking(PIEMRECOMPILERSTATE pReNative, uint32_t off)
453	{
454	#ifdef VBOX_STRICT
455	off = iemNativeEmitMarker(pReNative, off, 0x80000002);
456	#endif
457
458	/*
459	* This check must match the ones in the iem in iemGetTbFlagsForCurrentPc
460	* exactly:
461	*
462	* int64_t const offFromLim = (int64_t)pVCpu->cpum.GstCtx.cs.u32Limit - (int64_t)pVCpu->cpum.GstCtx.eip;
463	* if (offFromLim >= X86_PAGE_SIZE + 16 - (int32_t)(pVCpu->cpum.GstCtx.cs.u64Base & GUEST_PAGE_OFFSET_MASK))
464	* return fRet;
465	* return fRet \| IEMTB_F_CS_LIM_CHECKS;
466	*
467	*
468	* We need EIP, CS.LIM and CS.BASE here.
469	*/
470
471	/* Calculate the offFromLim first: */
472	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
473	kIemNativeGstRegUse_ReadOnly);
474	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
475	kIemNativeGstRegUse_ReadOnly);
476	uint8_t const idxRegLeft = iemNativeRegAllocTmp(pReNative, &off);
477
478	#ifdef RT_ARCH_ARM64
479	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
480	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegLeft, idxRegCsLim, idxRegPc);
481	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
482	#else
483	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegLeft, idxRegCsLim);
484	off = iemNativeEmitSubTwoGprs(pReNative, off, idxRegLeft, idxRegPc);
485	#endif
486
487	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
488	iemNativeRegFreeTmp(pReNative, idxRegPc);
489
490	/* Calculate the threshold level (right side). */
491	uint8_t const idxRegCsBase = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
492	kIemNativeGstRegUse_ReadOnly);
493	uint8_t const idxRegRight = iemNativeRegAllocTmp(pReNative, &off);
494
495	#ifdef RT_ARCH_ARM64
496	pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
497	Assert(Armv8A64ConvertImmRImmS2Mask32(11, 0) == GUEST_PAGE_OFFSET_MASK);
498	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegRight, idxRegCsBase, 11, 0, false /f64Bit/);
499	pu32CodeBuf[off++] = Armv8A64MkInstrNeg(idxRegRight);
500	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
501	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
502	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
503
504	#else
505	off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegRight, GUEST_PAGE_OFFSET_MASK);
506	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegRight, idxRegCsBase);
507	off = iemNativeEmitNegGpr(pReNative, off, idxRegRight);
508	off = iemNativeEmitAddGprImm(pReNative, off, idxRegRight, X86_PAGE_SIZE + 16);
509	#endif
510
511	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
512
513	/* Compare the two and jump out if we're too close to the limit. */
514	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
515	off = iemNativeEmitJlToNewLabel(pReNative, off, kIemNativeLabelType_NeedCsLimChecking);
516
517	iemNativeRegFreeTmp(pReNative, idxRegRight);
518	iemNativeRegFreeTmp(pReNative, idxRegLeft);
519	return off;
520	}
521
522
523
524	/**
525	* Macro that implements opcode (re-)checking.
526	*/
527	#define BODY_CHECK_OPCODES(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
528	RT_NOREF(a_cbInstr); \
529	off = iemNativeEmitBltInCheckOpcodes(pReNative, off, (a_pTb), (a_idxRange), (a_offRange))
530
531	#define LIVENESS_CHECK_OPCODES(a_pOutgoing) ((void)0)
532
533	#if 0 /* debugging aid */
534	bool g_fBpOnObsoletion = false;
535	# define BP_ON_OBSOLETION g_fBpOnObsoletion
536	#else
537	# define BP_ON_OBSOLETION 0
538	#endif
539
540	DECL_FORCE_INLINE(uint32_t)
541	iemNativeEmitBltInCheckOpcodes(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint16_t offRange)
542	{
543	Assert(idxRange < pTb->cRanges && pTb->cRanges <= RT_ELEMENTS(pTb->aRanges));
544	Assert(offRange < pTb->aRanges[idxRange].cbOpcodes);
545	#ifdef VBOX_STRICT
546	off = iemNativeEmitMarker(pReNative, off, 0x80000003);
547	#endif
548
549	uint32_t const idxLabelObsoleteTb = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ObsoleteTb);
550
551	/*
552	* Where to start and how much to compare.
553	*
554	* Looking at the ranges produced when r160746 was running a DOS VM with TB
555	* logging, the ranges can be anything from 1 byte to at least 0x197 bytes,
556	* with the 6, 5, 4, 7, 8, 40, 3, 2, 9 and 10 being the top 10 in the sample.
557	*
558	* The top 10 for the early boot phase of a 64-bit debian 9.4 VM: 5, 9, 8,
559	* 12, 10, 11, 6, 13, 15 and 16. Max 0x359 bytes. Same revision as above.
560	*/
561	uint16_t offPage = pTb->aRanges[idxRange].offPhysPage + offRange;
562	uint16_t cbLeft = pTb->aRanges[idxRange].cbOpcodes - offRange;
563	Assert(cbLeft > 0);
564	uint8_t const *pbOpcodes = &pTb->pabOpcodes[pTb->aRanges[idxRange].offOpcodes + offRange];
565	uint32_t offConsolidatedJump = UINT32_MAX;
566
567	#ifdef RT_ARCH_AMD64
568	/* AMD64/x86 offers a bunch of options. Smaller stuff will can be
569	completely inlined, for larger we use REPE CMPS. */
570	# define CHECK_OPCODES_CMP_IMMXX(a_idxReg, a_bOpcode) /* cost: 3 bytes */ do { \
571	pbCodeBuf[off++] = a_bOpcode; \
572	Assert(offPage < 127); \
573	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 7, a_idxReg); \
574	pbCodeBuf[off++] = RT_BYTE1(offPage); \
575	} while (0)
576
577	# define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
578	if (offConsolidatedJump != UINT32_MAX) \
579	{ \
580	int32_t const offDisp = (int32_t)offConsolidatedJump - (int32_t)(off + 2); \
581	Assert(offDisp >= -128); \
582	pbCodeBuf[off++] = 0x75; /* jnz near */ \
583	pbCodeBuf[off++] = (uint8_t)offDisp; \
584	} \
585	else \
586	{ \
587	pbCodeBuf[off++] = 0x74; /* jz near +5 */ \
588	pbCodeBuf[off++] = 0x05 + BP_ON_OBSOLETION; \
589	offConsolidatedJump = off; \
590	if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
591	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */ \
592	iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_Rel32, -4); \
593	pbCodeBuf[off++] = 0x00; \
594	pbCodeBuf[off++] = 0x00; \
595	pbCodeBuf[off++] = 0x00; \
596	pbCodeBuf[off++] = 0x00; \
597	} \
598	} while (0)
599
600	# define CHECK_OPCODES_CMP_IMM32(a_idxReg) /* cost: 3+4+2 = 9 */ do { \
601	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
602	pbCodeBuf[off++] = *pbOpcodes++; \
603	pbCodeBuf[off++] = *pbOpcodes++; \
604	pbCodeBuf[off++] = *pbOpcodes++; \
605	pbCodeBuf[off++] = *pbOpcodes++; \
606	cbLeft -= 4; \
607	offPage += 4; \
608	CHECK_OPCODES_CMP_JMP(); \
609	} while (0)
610
611	# define CHECK_OPCODES_CMP_IMM16(a_idxReg) /* cost: 1+3+2+2 = 8 */ do { \
612	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP; \
613	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
614	pbCodeBuf[off++] = *pbOpcodes++; \
615	pbCodeBuf[off++] = *pbOpcodes++; \
616	cbLeft -= 2; \
617	offPage += 2; \
618	CHECK_OPCODES_CMP_JMP(); \
619	} while (0)
620
621	# define CHECK_OPCODES_CMP_IMM8(a_idxReg) /* cost: 3+1+2 = 6 */ do { \
622	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x80); \
623	pbCodeBuf[off++] = *pbOpcodes++; \
624	cbLeft -= 1; \
625	offPage += 1; \
626	CHECK_OPCODES_CMP_JMP(); \
627	} while (0)
628
629	# define CHECK_OPCODES_CMPSX(a_bOpcode, a_cbToSubtract, a_bPrefix) /* cost: 2+2 = 4 */ do { \
630	if (a_bPrefix) \
631	pbCodeBuf[off++] = (a_bPrefix); \
632	pbCodeBuf[off++] = (a_bOpcode); \
633	CHECK_OPCODES_CMP_JMP(); \
634	cbLeft -= (a_cbToSubtract); \
635	} while (0)
636
637	# define CHECK_OPCODES_ECX_IMM(a_uValue) /* cost: 5 */ do { \
638	pbCodeBuf[off++] = 0xb8 + X86_GREG_xCX; \
639	pbCodeBuf[off++] = RT_BYTE1(a_uValue); \
640	pbCodeBuf[off++] = RT_BYTE2(a_uValue); \
641	pbCodeBuf[off++] = RT_BYTE3(a_uValue); \
642	pbCodeBuf[off++] = RT_BYTE4(a_uValue); \
643	} while (0)
644
645	if (cbLeft <= 24)
646	{
647	uint8_t const idxRegTmp = iemNativeRegAllocTmpEx(pReNative, &off,
648	( RT_BIT_32(X86_GREG_xAX)
649	\| RT_BIT_32(X86_GREG_xCX)
650	\| RT_BIT_32(X86_GREG_xDX)
651	\| RT_BIT_32(X86_GREG_xBX)
652	\| RT_BIT_32(X86_GREG_xSI)
653	\| RT_BIT_32(X86_GREG_xDI))
654	& ~IEMNATIVE_REG_FIXED_MASK); /* pick reg not requiring rex prefix */
655	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf));
656	if (offPage >= 128 - cbLeft)
657	{
658	off = iemNativeEmitAddGprImm(pReNative, off, idxRegTmp, offPage & ~(uint16_t)3);
659	offPage &= 3;
660	}
661
662	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 87 */);
663
664	if (cbLeft > 8)
665	switch (offPage & 3)
666	{
667	case 0:
668	break;
669	case 1: /* cost: 6 + 8 = 14 */
670	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
671	RT_FALL_THRU();
672	case 2: /* cost: 8 */
673	CHECK_OPCODES_CMP_IMM16(idxRegTmp);
674	break;
675	case 3: /* cost: 6 */
676	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
677	break;
678	}
679
680	while (cbLeft >= 4)
681	CHECK_OPCODES_CMP_IMM32(idxRegTmp); /* max iteration: 24/4 = 6; --> cost: 6 * 9 = 54 */
682
683	if (cbLeft >= 2)
684	CHECK_OPCODES_CMP_IMM16(idxRegTmp); /* cost: 8 */
685	if (cbLeft)
686	CHECK_OPCODES_CMP_IMM8(idxRegTmp); /* cost: 6 */
687
688	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
689	iemNativeRegFreeTmp(pReNative, idxRegTmp);
690	}
691	else
692	{
693	/* RDI = &pbInstrBuf[offPage] */
694	uint8_t const idxRegDi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xDI));
695	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegDi, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
696	if (offPage != 0)
697	off = iemNativeEmitAddGprImm(pReNative, off, idxRegDi, offPage);
698
699	/* RSI = pbOpcodes */
700	uint8_t const idxRegSi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xSI));
701	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegSi, (uintptr_t)pbOpcodes);
702
703	/* RCX = counts. */
704	uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
705
706	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /= 35/);
707
708	/** @todo profile and optimize this further. Maybe an idea to align by
709	* offPage if the two cannot be reconsidled. */
710	/* Align by the page offset, so that at least one of the accesses are naturally aligned. */
711	switch (offPage & 7) /* max cost: 10 */
712	{
713	case 0:
714	break;
715	case 1: /* cost: 3+4+3 = 10 */
716	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
717	RT_FALL_THRU();
718	case 2: /* cost: 4+3 = 7 */
719	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
720	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
721	break;
722	case 3: /* cost: 3+3 = 6 */
723	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
724	RT_FALL_THRU();
725	case 4: /* cost: 3 */
726	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
727	break;
728	case 5: /* cost: 3+4 = 7 */
729	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
730	RT_FALL_THRU();
731	case 6: /* cost: 4 */
732	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
733	break;
734	case 7: /* cost: 3 */
735	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
736	break;
737	}
738
739	/* Compare qwords: */
740	uint32_t const cQWords = cbLeft >> 3;
741	CHECK_OPCODES_ECX_IMM(cQWords); /* cost: 5 */
742
743	pbCodeBuf[off++] = X86_OP_PRF_REPZ; /* cost: 5 */
744	CHECK_OPCODES_CMPSX(0xa7, 0, X86_OP_REX_W);
745	cbLeft &= 7;
746
747	if (cbLeft & 4)
748	CHECK_OPCODES_CMPSX(0xa7, 4, 0); /* cost: 3 */
749	if (cbLeft & 2)
750	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP); /* cost: 4 */
751	if (cbLeft & 1)
752	CHECK_OPCODES_CMPSX(0xa6, 1, 0); /* cost: 3 */
753
754	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
755	iemNativeRegFreeTmp(pReNative, idxRegCx);
756	iemNativeRegFreeTmp(pReNative, idxRegSi);
757	iemNativeRegFreeTmp(pReNative, idxRegDi);
758	}
759
760	#elif defined(RT_ARCH_ARM64)
761	/* We need pbInstrBuf in a register, whatever we do. */
762	uint8_t const idxRegSrc1Ptr = iemNativeRegAllocTmp(pReNative, &off);
763	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegSrc1Ptr, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
764
765	/* We also need at least one more register for holding bytes & words we
766	load via pbInstrBuf. */
767	uint8_t const idxRegSrc1Val = iemNativeRegAllocTmp(pReNative, &off);
768
769	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
770
771	/* One byte compare can be done with the opcode byte as an immediate. We'll
772	do this to uint16_t align src1. */
773	bool fPendingJmp = RT_BOOL(offPage & 1);
774	if (fPendingJmp)
775	{
776	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
777	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes++, false /f64Bit*/);
778	offPage += 1;
779	cbLeft -= 1;
780	}
781
782	if (cbLeft > 0)
783	{
784	/* We need a register for holding the opcode bytes we're comparing with,
785	as CCMP only has a 5-bit immediate form and thus cannot hold bytes. */
786	uint8_t const idxRegSrc2Val = iemNativeRegAllocTmp(pReNative, &off);
787
788	/* Word (uint32_t) aligning the src1 pointer is best done using a 16-bit constant load. */
789	if ((offPage & 3) && cbLeft >= 2)
790	{
791	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 2);
792	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
793	if (fPendingJmp)
794	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
795	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
796	else
797	{
798	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
799	fPendingJmp = true;
800	}
801	pbOpcodes += 2;
802	offPage += 2;
803	cbLeft -= 2;
804	}
805
806	/* DWord (uint64_t) aligning the src2 pointer. We use a 32-bit constant here for simplicitly. */
807	if ((offPage & 7) && cbLeft >= 4)
808	{
809	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 4);
810	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
811	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
812	if (fPendingJmp)
813	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
814	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
815	else
816	{
817	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
818	fPendingJmp = true;
819	}
820	pbOpcodes += 4;
821	offPage += 4;
822	cbLeft -= 4;
823	}
824
825	/*
826	* If we've got 16 bytes or more left, switch to memcmp-style.
827	*/
828	if (cbLeft >= 16)
829	{
830	/* We need a pointer to the copy of the original opcode bytes. */
831	uint8_t const idxRegSrc2Ptr = iemNativeRegAllocTmp(pReNative, &off);
832	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Ptr, (uintptr_t)pbOpcodes);
833
834	/* If there are more than 32 bytes to compare we create a loop, for
835	which we'll need a loop register. */
836	if (cbLeft >= 64)
837	{
838	if (fPendingJmp)
839	{
840	iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
841	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
842	fPendingJmp = false;
843	}
844
845	uint8_t const idxRegLoop = iemNativeRegAllocTmp(pReNative, &off);
846	uint16_t const cLoops = cbLeft / 32;
847	cbLeft = cbLeft % 32;
848	pbOpcodes += cLoops * 32;
849	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegLoop, cLoops);
850
851	if (offPage != 0) /** @todo optimize out this instruction. */
852	{
853	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, offPage);
854	offPage = 0;
855	}
856
857	uint32_t const offLoopStart = off;
858	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 0);
859	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 0);
860	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
861
862	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 1);
863	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 1);
864	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
865	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
866
867	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 2);
868	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 2);
869	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
870	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
871
872	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 3);
873	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 3);
874	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
875	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
876
877	iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
878	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
879
880	/* Advance and loop. */
881	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, 0x20);
882	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc2Ptr, idxRegSrc2Ptr, 0x20);
883	pu32CodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxRegLoop, idxRegLoop, 1, false /f64Bit/, true /fSetFlags/);
884	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, (int32_t)offLoopStart - (int32_t)off);
885
886	iemNativeRegFreeTmp(pReNative, idxRegLoop);
887	}
888
889	/* Deal with any remaining dwords (uint64_t). There can be up to
890	three if we looped and four if we didn't. */
891	uint32_t offSrc2 = 0;
892	while (cbLeft >= 8)
893	{
894	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val,
895	idxRegSrc1Ptr, offPage / 8);
896	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val,
897	idxRegSrc2Ptr, offSrc2 / 8);
898	if (fPendingJmp)
899	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
900	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
901	else
902	{
903	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
904	fPendingJmp = true;
905	}
906	pbOpcodes += 8;
907	offPage += 8;
908	offSrc2 += 8;
909	cbLeft -= 8;
910	}
911
912	iemNativeRegFreeTmp(pReNative, idxRegSrc2Ptr);
913	/* max cost thus far: memcmp-loop=43 vs memcmp-no-loop=30 */
914	}
915	/*
916	* Otherwise, we compare with constants and merge with the general mop-up.
917	*/
918	else
919	{
920	while (cbLeft >= 8)
921	{
922	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr,
923	offPage / 8);
924	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Val,
925	RT_MAKE_U64_FROM_MSB_U8(pbOpcodes[7], pbOpcodes[6], pbOpcodes[5], pbOpcodes[4],
926	pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
927	if (fPendingJmp)
928	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
929	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, true /f64Bit/);
930	else
931	{
932	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, true /f64Bit/);
933	fPendingJmp = true;
934	}
935	pbOpcodes += 8;
936	offPage += 8;
937	cbLeft -= 8;
938	}
939	/* max cost thus far: 21 */
940	}
941
942	/* Deal with any remaining bytes (7 or less). */
943	Assert(cbLeft < 8);
944	if (cbLeft >= 4)
945	{
946	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr,
947	offPage / 4);
948	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
949	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
950	if (fPendingJmp)
951	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
952	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
953	else
954	{
955	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
956	fPendingJmp = true;
957	}
958	pbOpcodes += 4;
959	offPage += 4;
960	cbLeft -= 4;
961
962	}
963
964	if (cbLeft >= 2)
965	{
966	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr,
967	offPage / 2);
968	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
969	if (fPendingJmp)
970	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
971	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
972	else
973	{
974	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
975	fPendingJmp = true;
976	}
977	pbOpcodes += 2;
978	offPage += 2;
979	cbLeft -= 2;
980	}
981
982	if (cbLeft > 0)
983	{
984	Assert(cbLeft == 1);
985	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
986	if (fPendingJmp)
987	{
988	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, pbOpcodes[0]);
989	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
990	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
991	}
992	else
993	{
994	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes[0], false /f64Bit/);
995	fPendingJmp = true;
996	}
997	pbOpcodes += 1;
998	offPage += 1;
999	cbLeft -= 1;
1000	}
1001
1002	iemNativeRegFreeTmp(pReNative, idxRegSrc2Val);
1003	}
1004	Assert(cbLeft == 0);
1005
1006	/*
1007	* Finally, the branch on difference.
1008	*/
1009	if (fPendingJmp)
1010	{
1011	iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
1012	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
1013	}
1014	RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump, idxLabelObsoleteTb);
1015
1016	/* max costs: memcmp-loop=54; memcmp-no-loop=41; only-src1-ptr=32 */
1017	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1018	iemNativeRegFreeTmp(pReNative, idxRegSrc1Val);
1019	iemNativeRegFreeTmp(pReNative, idxRegSrc1Ptr);
1020
1021	#else
1022	# error "Port me"
1023	#endif
1024	return off;
1025	}
1026
1027
1028	/** Duplicated in IEMAllThrdFuncsBltIn.cpp. */
1029	DECL_FORCE_INLINE(RTGCPHYS) iemTbGetRangePhysPageAddr(PCIEMTB pTb, uint8_t idxRange)
1030	{
1031	Assert(idxRange < RT_MIN(pTb->cRanges, RT_ELEMENTS(pTb->aRanges)));
1032	uint8_t const idxPage = pTb->aRanges[idxRange].idxPhysPage;
1033	Assert(idxPage <= RT_ELEMENTS(pTb->aGCPhysPages));
1034	if (idxPage == 0)
1035	return pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1036	Assert(!(pTb->aGCPhysPages[idxPage - 1] & GUEST_PAGE_OFFSET_MASK));
1037	return pTb->aGCPhysPages[idxPage - 1];
1038	}
1039
1040
1041	/**
1042	* Macro that implements PC check after a conditional branch.
1043	*/
1044	#define BODY_CHECK_PC_AFTER_BRANCH(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
1045	RT_NOREF(a_cbInstr); \
1046	off = iemNativeEmitBltInCheckPcAfterBranch(pReNative, off, a_pTb, a_idxRange, a_offRange)
1047
1048	#define LIVENESS_CHECK_PC_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1049	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1050	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1051	else do { } while (0)
1052
1053	DECL_FORCE_INLINE(uint32_t)
1054	iemNativeEmitBltInCheckPcAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb,
1055	uint8_t idxRange, uint16_t offRange)
1056	{
1057	#ifdef VBOX_STRICT
1058	off = iemNativeEmitMarker(pReNative, off, 0x80000004);
1059	#endif
1060
1061	/*
1062	* The GCPhysRangePageWithOffset value in the threaded function is a fixed
1063	* constant for us here.
1064	*
1065	* We can pretend that iem.s.cbInstrBufTotal is X86_PAGE_SIZE here, because
1066	* it serves no purpose as a CS.LIM, if that's needed we've just performed
1067	* it, and as long as we don't implement code TLB reload code here there is
1068	* no point in checking that the TLB data we're using is still valid.
1069	*
1070	* What we to do is.
1071	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1072	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1073	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1074	* we're in the wrong spot and need to find a new TB.
1075	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1076	* GCPhysRangePageWithOffset constant mentioned above.
1077	*
1078	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1079	* in 64-bit code or flat 32-bit.
1080	*/
1081
1082	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1083	the temp register, so we don't accidentally clobber something we'll be
1084	needing again immediately. This is why we get idxRegCsBase here. */
1085	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1086	kIemNativeGstRegUse_ReadOnly);
1087	uint8_t const idxRegCsBase = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX
1088	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1089	kIemNativeGstRegUse_ReadOnly);
1090
1091	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
1092
1093	#ifdef VBOX_STRICT
1094	/* Do assertions before idxRegTmp contains anything. */
1095	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1096	# ifdef RT_ARCH_AMD64
1097	{
1098	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1099	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1100	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1101	{
1102	/* cmp r/m64, imm8 */
1103	pbCodeBuf[off++] = X86_OP_REX_W;
1104	pbCodeBuf[off++] = 0x83;
1105	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1106	pbCodeBuf[off++] = 0;
1107	/* je rel8 */
1108	pbCodeBuf[off++] = 0x74;
1109	pbCodeBuf[off++] = 1;
1110	/* int3 */
1111	pbCodeBuf[off++] = 0xcc;
1112
1113	}
1114
1115	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1116	/* test r/m64, imm32 */
1117	pbCodeBuf[off++] = X86_OP_REX_W;
1118	pbCodeBuf[off++] = 0xf7;
1119	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1120	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1121	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1122	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1123	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1124	/* jz rel8 */
1125	pbCodeBuf[off++] = 0x74;
1126	pbCodeBuf[off++] = 1;
1127	/* int3 */
1128	pbCodeBuf[off++] = 0xcc;
1129	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1130	}
1131	# else
1132
1133	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1134	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1135	{
1136	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1137	# ifdef RT_ARCH_ARM64
1138	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1139	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1140	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2004);
1141	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1142	# else
1143	# error "Port me!"
1144	# endif
1145	}
1146	# endif
1147
1148	#endif /* VBOX_STRICT */
1149
1150	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1151	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1152	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1153	{
1154	#ifdef RT_ARCH_ARM64
1155	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1156	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1157	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1158	#else
1159	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1160	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1161	#endif
1162	}
1163	else
1164	{
1165	#ifdef RT_ARCH_ARM64
1166	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1167	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1168	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1169	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1170	#else
1171	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1172	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1173	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1174	#endif
1175	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1176	}
1177	iemNativeRegFreeTmp(pReNative, idxRegPc);
1178
1179	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
1180	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1181	off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
1182
1183	/* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
1184	#ifdef RT_ARCH_AMD64
1185	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1186	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1187	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1188	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1189	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1190
1191	#elif defined(RT_ARCH_ARM64)
1192	uint8_t const idxRegTmp2 = iemNativeRegAllocTmp(pReNative, &off);
1193
1194	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1195	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1196	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1197	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1198
1199	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1200	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1201	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1202	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1203	# endif
1204	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1205	#else
1206	# error "Port me"
1207	#endif
1208
1209	RTGCPHYS const GCPhysRangePageWithOffset = ( iemTbGetRangePhysPageAddr(pTb, idxRange)
1210	\| pTb->aRanges[idxRange].offPhysPage)
1211	+ offRange;
1212	off = iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
1213	kIemNativeLabelType_CheckBranchMiss);
1214
1215	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1216	return off;
1217	}
1218
1219
1220	/**
1221	* Macro that implements TLB loading and updating pbInstrBuf updating for an
1222	* instruction crossing into a new page.
1223	*
1224	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1225	*/
1226	#define BODY_LOAD_TLB_FOR_NEW_PAGE(a_pTb, a_offInstr, a_idxRange, a_cbInstr) \
1227	RT_NOREF(a_cbInstr); \
1228	off = iemNativeEmitBltLoadTlbForNewPage(pReNative, off, pTb, a_idxRange, a_offInstr)
1229
1230	#define LIVENESS_LOAD_TLB_FOR_NEW_PAGE(a_pOutgoing, a_pCallEntry) \
1231	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1232	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1233	else do { } while (0)
1234
1235	DECL_FORCE_INLINE(uint32_t)
1236	iemNativeEmitBltLoadTlbForNewPage(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint8_t offInstr)
1237	{
1238	#ifdef VBOX_STRICT
1239	off = iemNativeEmitMarker(pReNative, off, 0x80000005);
1240	#endif
1241
1242	/*
1243	* Define labels and allocate the register for holding the GCPhys of the new page.
1244	*/
1245	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1246	uint32_t const idxRegGCPhys = iemNativeRegAllocTmp(pReNative, &off);
1247	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, IEM_F_MODE_X86_IS_FLAT(pReNative->fExec), &off);
1248	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1249	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1250	: UINT32_MAX;
1251
1252	//off = iemNativeEmitBrk(pReNative, off, 0x1111);
1253
1254	/*
1255	* Jump to the TLB lookup code.
1256	*/
1257	if (!TlbState.fSkip)
1258	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1259
1260	/*
1261	* TlbMiss:
1262	*
1263	* Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
1264	*/
1265	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1266
1267	/* Save variables in volatile registers. */
1268	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegGCPhys);
1269	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1270
1271	/* IEMNATIVE_CALL_ARG1_GREG = offInstr */
1272	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offInstr);
1273
1274	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1275	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1276
1277	/* Done setting up parameters, make the call. */
1278	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMissWithOff);
1279
1280	/* Move the result to the right register. */
1281	if (idxRegGCPhys != IEMNATIVE_CALL_RET_GREG)
1282	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegGCPhys, IEMNATIVE_CALL_RET_GREG);
1283
1284	/* Restore variables and guest shadow registers to volatile registers. */
1285	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1286	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows(true /fCode/));
1287
1288	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1289	if (!TlbState.fSkip)
1290	{
1291	/* end of TlbMiss - Jump to the done label. */
1292	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1293	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1294
1295	/*
1296	* TlbLookup:
1297	*/
1298	off = iemNativeEmitTlbLookup<false>(pReNative, off, &TlbState,
1299	IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX : X86_SREG_CS,
1300	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1301	idxLabelTlbLookup, idxLabelTlbMiss, idxRegGCPhys, offInstr);
1302
1303	# ifdef VBOX_WITH_STATISTICS
1304	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1305	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPageWithOffset));
1306	# endif
1307
1308	/*
1309	* TlbDone:
1310	*/
1311	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1312	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1313	}
1314	#else
1315	RT_NOREF(idxLabelTlbMiss);
1316	#endif
1317
1318	/*
1319	* Now check the physical address of the page matches the expected one.
1320	*/
1321	RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
1322	off = iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(pReNative, off, idxRegGCPhys, GCPhysNewPage,
1323	kIemNativeLabelType_ObsoleteTb);
1324
1325	iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
1326	return off;
1327	}
1328
1329
1330	/**
1331	* Macro that implements TLB loading and updating pbInstrBuf updating when
1332	* branching or when crossing a page on an instruction boundrary.
1333	*
1334	* This differs from BODY_LOAD_TLB_FOR_NEW_PAGE in that it will first check if
1335	* it is an inter-page branch and also check the page offset.
1336	*
1337	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1338	*/
1339	#define BODY_LOAD_TLB_AFTER_BRANCH(a_pTb, a_idxRange, a_cbInstr) \
1340	RT_NOREF(a_cbInstr); \
1341	off = iemNativeEmitBltLoadTlbAfterBranch(pReNative, off, pTb, a_idxRange)
1342
1343	#define LIVENESS_LOAD_TLB_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1344	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1345	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1346	else do { } while (0)
1347
1348	DECL_FORCE_INLINE(uint32_t)
1349	iemNativeEmitBltLoadTlbAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange)
1350	{
1351	#ifdef VBOX_STRICT
1352	off = iemNativeEmitMarker(pReNative, off, 0x80000006);
1353	#endif
1354
1355	/*
1356	* Define labels and allocate the register for holding the GCPhys of the new page.
1357	*/
1358	uint32_t const idxLabelCheckBranchMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckBranchMiss);
1359	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1360	RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(pTb, idxRange)
1361	\| pTb->aRanges[idxRange].offPhysPage;
1362
1363	/*
1364	*
1365	* First check if RIP is within the current code.
1366	*
1367	* This is very similar to iemNativeEmitBltInCheckPcAfterBranch, the only
1368	* difference is what we do when stuff doesn't match up.
1369	*
1370	* What we to do is.
1371	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1372	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1373	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1374	* we need to retranslate RIP via the TLB.
1375	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1376	* GCPhysRangePageWithOffset constant mentioned above.
1377	*
1378	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1379	* in 64-bit code or flat 32-bit.
1380	*
1381	*/
1382
1383	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1384	the temp register, so we don't accidentally clobber something we'll be
1385	needing again immediately. This is why we get idxRegCsBase here.
1386	Update: We share registers with the TlbState, as the TLB code path has
1387	little in common with the rest of the code. */
1388	bool const fIsFlat = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec);
1389	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, fIsFlat, &off);
1390	uint8_t const idxRegPc = !TlbState.fSkip ? TlbState.idxRegPtr
1391	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1392	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1393	uint8_t const idxRegCsBase = !TlbState.fSkip \|\| fIsFlat ? TlbState.idxRegSegBase
1394	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1395	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1396
1397	uint8_t const idxRegTmp = !TlbState.fSkip ? TlbState.idxReg1 : iemNativeRegAllocTmp(pReNative, &off);
1398	uint8_t const idxRegTmp2 = !TlbState.fSkip ? TlbState.idxReg2 : iemNativeRegAllocTmp(pReNative, &off);
1399	uint8_t const idxRegDummy = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
1400
1401	#ifdef VBOX_STRICT
1402	/* Do assertions before idxRegTmp contains anything. */
1403	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1404	# ifdef RT_ARCH_AMD64
1405	{
1406	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1407	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1408	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1409	{
1410	/* cmp r/m64, imm8 */
1411	pbCodeBuf[off++] = X86_OP_REX_W;
1412	pbCodeBuf[off++] = 0x83;
1413	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1414	pbCodeBuf[off++] = 0;
1415	/* je rel8 */
1416	pbCodeBuf[off++] = 0x74;
1417	pbCodeBuf[off++] = 1;
1418	/* int3 */
1419	pbCodeBuf[off++] = 0xcc;
1420
1421	}
1422
1423	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1424	/* test r/m64, imm32 */
1425	pbCodeBuf[off++] = X86_OP_REX_W;
1426	pbCodeBuf[off++] = 0xf7;
1427	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1428	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1429	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1430	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1431	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1432	/* jz rel8 */
1433	pbCodeBuf[off++] = 0x74;
1434	pbCodeBuf[off++] = 1;
1435	/* int3 */
1436	pbCodeBuf[off++] = 0xcc;
1437	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1438	}
1439	# else
1440
1441	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1442	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1443	{
1444	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1445	# ifdef RT_ARCH_ARM64
1446	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1447	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1448	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2006);
1449	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1450	# else
1451	# error "Port me!"
1452	# endif
1453	}
1454	# endif
1455
1456	#endif /* VBOX_STRICT */
1457
1458	/* Because we're lazy, we'll jump back here to recalc 'off' and share the
1459	GCPhysRangePageWithOffset check. This is a little risky, so we use the
1460	2nd register to check if we've looped more than once already.*/
1461	off = iemNativeEmitGprZero(pReNative, off, idxRegTmp2);
1462
1463	uint32_t const offLabelRedoChecks = off;
1464
1465	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1466	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1467	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1468	{
1469	#ifdef RT_ARCH_ARM64
1470	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1471	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1472	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1473	#else
1474	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1475	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1476	#endif
1477	}
1478	else
1479	{
1480	#ifdef RT_ARCH_ARM64
1481	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1482	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1483	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1484	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1485	#else
1486	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1487	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1488	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1489	#endif
1490	}
1491
1492	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal.
1493	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll jump to the TLB loading if this fails. */
1494	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1495	uint32_t const offFixedJumpToTlbLoad = off;
1496	off = iemNativeEmitJaToFixed(pReNative, off, off /* (ASSUME ja rel8 suffices) */);
1497
1498	/* 4a. Add iem.s.GCPhysInstrBuf to off ... */
1499	#ifdef RT_ARCH_AMD64
1500	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1501	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1502	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1503	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1504	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1505
1506	#elif defined(RT_ARCH_ARM64)
1507
1508	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1509	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1510	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1511	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1512
1513	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1514	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1515	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1516	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1517	# endif
1518	#else
1519	# error "Port me"
1520	#endif
1521
1522	/* 4b. ... and compare with GCPhysRangePageWithOffset.
1523
1524	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll have to be more
1525	careful and avoid implicit temporary register usage here.
1526
1527	Unlike the threaded version of this code, we do not obsolete TBs here to
1528	reduce the code size and because indirect calls may legally end at the
1529	same offset in two different pages depending on the program state. */
1530	/** @todo synch the threaded BODY_LOAD_TLB_AFTER_BRANCH version with this. */
1531	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
1532	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
1533	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelCheckBranchMiss);
1534	uint32_t const offFixedJumpToEnd = off;
1535	off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
1536
1537	/*
1538	* TlbLoad:
1539	*
1540	* First we try to go via the TLB.
1541	*/
1542	iemNativeFixupFixedJump(pReNative, offFixedJumpToTlbLoad, off);
1543
1544	/* Check that we haven't been here before. */
1545	off = iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, idxRegTmp2, false /f64Bit/, idxLabelCheckBranchMiss);
1546
1547	/* Jump to the TLB lookup code. */
1548	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1549	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1550	: UINT32_MAX;
1551	//off = iemNativeEmitBrk(pReNative, off, 0x1234);
1552	if (!TlbState.fSkip)
1553	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1554
1555	/*
1556	* TlbMiss:
1557	*
1558	* Call iemNativeHlpMemCodeNewPageTlbMiss to do the work.
1559	*/
1560	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1561	RT_NOREF(idxLabelTlbMiss);
1562
1563	/* Save variables in volatile registers. */
1564	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegTmp) \| RT_BIT_32(idxRegTmp2)
1565	\| (idxRegDummy != UINT8_MAX ? RT_BIT_32(idxRegDummy) : 0);
1566	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1567
1568	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1569	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1570
1571	/* Done setting up parameters, make the call. */
1572	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMiss);
1573
1574	/* Restore variables and guest shadow registers to volatile registers. */
1575	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1576	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off,
1577	TlbState.getActiveRegsWithShadows()
1578	\| RT_BIT_32(idxRegPc)
1579	\| (idxRegCsBase != UINT8_MAX ? RT_BIT_32(idxRegCsBase) : 0));
1580
1581	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1582	if (!TlbState.fSkip)
1583	{
1584	/* end of TlbMiss - Jump to the done label. */
1585	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1586	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1587
1588	/*
1589	* TlbLookup:
1590	*/
1591	off = iemNativeEmitTlbLookup<false, true>(pReNative, off, &TlbState, fIsFlat ? UINT8_MAX : X86_SREG_CS,
1592	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1593	idxLabelTlbLookup, idxLabelTlbMiss, idxRegDummy);
1594
1595	# ifdef VBOX_WITH_STATISTICS
1596	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1597	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPage));
1598	# endif
1599
1600	/*
1601	* TlbDone:
1602	*/
1603	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1604	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1605	}
1606	#else
1607	RT_NOREF(idxLabelTlbMiss);
1608	#endif
1609
1610	/* Jmp back to the start and redo the checks. */
1611	off = iemNativeEmitLoadGpr8Imm(pReNative, off, idxRegTmp2, 1); /* indicate that we've looped once already */
1612	off = iemNativeEmitJmpToFixed(pReNative, off, offLabelRedoChecks);
1613
1614	/*
1615	* End:
1616	*
1617	* The end.
1618	*/
1619	iemNativeFixupFixedJump(pReNative, offFixedJumpToEnd, off);
1620
1621	if (!TlbState.fSkip)
1622	iemNativeRegFreeTmp(pReNative, idxRegDummy);
1623	else
1624	{
1625	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1626	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1627	iemNativeRegFreeTmp(pReNative, idxRegPc);
1628	if (idxRegCsBase != UINT8_MAX)
1629	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1630	}
1631	return off;
1632	}
1633
1634
1635	#ifdef BODY_CHECK_CS_LIM
1636	/**
1637	* Built-in function that checks the EIP/IP + uParam0 is within CS.LIM,
1638	* raising a \#GP(0) if this isn't the case.
1639	*/
1640	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLim)
1641	{
1642	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1643	BODY_SET_CUR_INSTR();
1644	BODY_FLUSH_PENDING_WRITES();
1645	BODY_CHECK_CS_LIM(cbInstr);
1646	return off;
1647	}
1648
1649	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLim)
1650	{
1651	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1652	LIVENESS_CHECK_CS_LIM(pOutgoing);
1653	RT_NOREF(pCallEntry);
1654	}
1655	#endif
1656
1657
1658	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_CS_LIM)
1659	/**
1660	* Built-in function for re-checking opcodes and CS.LIM after an instruction
1661	* that may have modified them.
1662	*/
1663	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodes)
1664	{
1665	PCIEMTB const pTb = pReNative->pTbOrg;
1666	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1667	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1668	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1669	BODY_SET_CUR_INSTR();
1670	BODY_FLUSH_PENDING_WRITES();
1671	BODY_CHECK_CS_LIM(cbInstr);
1672	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1673	return off;
1674	}
1675
1676	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodes)
1677	{
1678	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1679	LIVENESS_CHECK_CS_LIM(pOutgoing);
1680	LIVENESS_CHECK_OPCODES(pOutgoing);
1681	RT_NOREF(pCallEntry);
1682	}
1683	#endif
1684
1685
1686	#if defined(BODY_CHECK_OPCODES)
1687	/**
1688	* Built-in function for re-checking opcodes after an instruction that may have
1689	* modified them.
1690	*/
1691	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodes)
1692	{
1693	PCIEMTB const pTb = pReNative->pTbOrg;
1694	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1695	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1696	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1697	BODY_SET_CUR_INSTR();
1698	BODY_FLUSH_PENDING_WRITES();
1699	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1700	return off;
1701	}
1702
1703	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodes)
1704	{
1705	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1706	LIVENESS_CHECK_OPCODES(pOutgoing);
1707	RT_NOREF(pCallEntry);
1708	}
1709	#endif
1710
1711
1712	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1713	/**
1714	* Built-in function for re-checking opcodes and considering the need for CS.LIM
1715	* checking after an instruction that may have modified them.
1716	*/
1717	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesConsiderCsLim)
1718	{
1719	PCIEMTB const pTb = pReNative->pTbOrg;
1720	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1721	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1722	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1723	BODY_SET_CUR_INSTR();
1724	BODY_FLUSH_PENDING_WRITES();
1725	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1726	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1727	return off;
1728	}
1729
1730	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesConsiderCsLim)
1731	{
1732	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1733	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1734	LIVENESS_CHECK_OPCODES(pOutgoing);
1735	RT_NOREF(pCallEntry);
1736	}
1737	#endif
1738
1739
1740	/*
1741	* Post-branching checkers.
1742	*/
1743
1744	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1745	/**
1746	* Built-in function for checking CS.LIM, checking the PC and checking opcodes
1747	* after conditional branching within the same page.
1748	*
1749	* @see iemThreadedFunc_BltIn_CheckPcAndOpcodes
1750	*/
1751	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1752	{
1753	PCIEMTB const pTb = pReNative->pTbOrg;
1754	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1755	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1756	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1757	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1758	BODY_SET_CUR_INSTR();
1759	BODY_FLUSH_PENDING_WRITES();
1760	BODY_CHECK_CS_LIM(cbInstr);
1761	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1762	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1763	//LogFunc(("okay\n"));
1764	return off;
1765	}
1766
1767	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1768	{
1769	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1770	LIVENESS_CHECK_CS_LIM(pOutgoing);
1771	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1772	LIVENESS_CHECK_OPCODES(pOutgoing);
1773	RT_NOREF(pCallEntry);
1774	}
1775	#endif
1776
1777
1778	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH)
1779	/**
1780	* Built-in function for checking the PC and checking opcodes after conditional
1781	* branching within the same page.
1782	*
1783	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1784	*/
1785	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodes)
1786	{
1787	PCIEMTB const pTb = pReNative->pTbOrg;
1788	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1789	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1790	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1791	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1792	BODY_SET_CUR_INSTR();
1793	BODY_FLUSH_PENDING_WRITES();
1794	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1795	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1796	//LogFunc(("okay\n"));
1797	return off;
1798	}
1799
1800	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodes)
1801	{
1802	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1803	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1804	LIVENESS_CHECK_OPCODES(pOutgoing);
1805	RT_NOREF(pCallEntry);
1806	}
1807	#endif
1808
1809
1810	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1811	/**
1812	* Built-in function for checking the PC and checking opcodes and considering
1813	* the need for CS.LIM checking after conditional branching within the same
1814	* page.
1815	*
1816	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1817	*/
1818	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1819	{
1820	PCIEMTB const pTb = pReNative->pTbOrg;
1821	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1822	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1823	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1824	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1825	BODY_SET_CUR_INSTR();
1826	BODY_FLUSH_PENDING_WRITES();
1827	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1828	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1829	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1830	//LogFunc(("okay\n"));
1831	return off;
1832	}
1833
1834	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1835	{
1836	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1837	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1838	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1839	LIVENESS_CHECK_OPCODES(pOutgoing);
1840	RT_NOREF(pCallEntry);
1841	}
1842	#endif
1843
1844
1845	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1846	/**
1847	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
1848	* transitioning to a different code page.
1849	*
1850	* The code page transition can either be natural over onto the next page (with
1851	* the instruction starting at page offset zero) or by means of branching.
1852	*
1853	* @see iemThreadedFunc_BltIn_CheckOpcodesLoadingTlb
1854	*/
1855	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1856	{
1857	PCIEMTB const pTb = pReNative->pTbOrg;
1858	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1859	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1860	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1861	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1862	BODY_SET_CUR_INSTR();
1863	BODY_FLUSH_PENDING_WRITES();
1864	BODY_CHECK_CS_LIM(cbInstr);
1865	Assert(offRange == 0);
1866	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1867	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1868	//LogFunc(("okay\n"));
1869	return off;
1870	}
1871
1872	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1873	{
1874	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1875	LIVENESS_CHECK_CS_LIM(pOutgoing);
1876	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1877	LIVENESS_CHECK_OPCODES(pOutgoing);
1878	RT_NOREF(pCallEntry);
1879	}
1880	#endif
1881
1882
1883	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH)
1884	/**
1885	* Built-in function for loading TLB and checking opcodes when transitioning to
1886	* a different code page.
1887	*
1888	* The code page transition can either be natural over onto the next page (with
1889	* the instruction starting at page offset zero) or by means of branching.
1890	*
1891	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1892	*/
1893	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlb)
1894	{
1895	PCIEMTB const pTb = pReNative->pTbOrg;
1896	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1897	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1898	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1899	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1900	BODY_SET_CUR_INSTR();
1901	BODY_FLUSH_PENDING_WRITES();
1902	Assert(offRange == 0);
1903	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1904	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1905	//LogFunc(("okay\n"));
1906	return off;
1907	}
1908
1909	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlb)
1910	{
1911	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1912	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1913	LIVENESS_CHECK_OPCODES(pOutgoing);
1914	RT_NOREF(pCallEntry);
1915	}
1916	#endif
1917
1918
1919	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1920	/**
1921	* Built-in function for loading TLB and checking opcodes and considering the
1922	* need for CS.LIM checking when transitioning to a different code page.
1923	*
1924	* The code page transition can either be natural over onto the next page (with
1925	* the instruction starting at page offset zero) or by means of branching.
1926	*
1927	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1928	*/
1929	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1930	{
1931	PCIEMTB const pTb = pReNative->pTbOrg;
1932	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1933	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1934	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1935	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1936	BODY_SET_CUR_INSTR();
1937	BODY_FLUSH_PENDING_WRITES();
1938	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1939	Assert(offRange == 0);
1940	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1941	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1942	//LogFunc(("okay\n"));
1943	return off;
1944	}
1945
1946	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1947	{
1948	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1949	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1950	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1951	LIVENESS_CHECK_OPCODES(pOutgoing);
1952	RT_NOREF(pCallEntry);
1953	}
1954	#endif
1955
1956
1957
1958	/*
1959	* Natural page crossing checkers.
1960	*/
1961
1962	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
1963	/**
1964	* Built-in function for checking CS.LIM, loading TLB and checking opcodes on
1965	* both pages when transitioning to a different code page.
1966	*
1967	* This is used when the previous instruction requires revalidation of opcodes
1968	* bytes and the current instruction stries a page boundrary with opcode bytes
1969	* in both the old and new page.
1970	*
1971	* @see iemThreadedFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb
1972	*/
1973	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
1974	{
1975	PCIEMTB const pTb = pReNative->pTbOrg;
1976	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1977	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
1978	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
1979	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
1980	uint32_t const idxRange2 = idxRange1 + 1;
1981	BODY_SET_CUR_INSTR();
1982	BODY_FLUSH_PENDING_WRITES();
1983	BODY_CHECK_CS_LIM(cbInstr);
1984	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
1985	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
1986	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
1987	return off;
1988	}
1989
1990	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
1991	{
1992	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1993	LIVENESS_CHECK_CS_LIM(pOutgoing);
1994	LIVENESS_CHECK_OPCODES(pOutgoing);
1995	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
1996	RT_NOREF(pCallEntry);
1997	}
1998	#endif
1999
2000
2001	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2002	/**
2003	* Built-in function for loading TLB and checking opcodes on both pages when
2004	* transitioning to a different code page.
2005	*
2006	* This is used when the previous instruction requires revalidation of opcodes
2007	* bytes and the current instruction stries a page boundrary with opcode bytes
2008	* in both the old and new page.
2009	*
2010	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2011	*/
2012	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2013	{
2014	PCIEMTB const pTb = pReNative->pTbOrg;
2015	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2016	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2017	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2018	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2019	uint32_t const idxRange2 = idxRange1 + 1;
2020	BODY_SET_CUR_INSTR();
2021	BODY_FLUSH_PENDING_WRITES();
2022	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2023	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2024	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2025	return off;
2026	}
2027
2028	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2029	{
2030	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2031	LIVENESS_CHECK_OPCODES(pOutgoing);
2032	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2033	RT_NOREF(pCallEntry);
2034	}
2035	#endif
2036
2037
2038	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2039	/**
2040	* Built-in function for loading TLB and checking opcodes on both pages and
2041	* considering the need for CS.LIM checking when transitioning to a different
2042	* code page.
2043	*
2044	* This is used when the previous instruction requires revalidation of opcodes
2045	* bytes and the current instruction stries a page boundrary with opcode bytes
2046	* in both the old and new page.
2047	*
2048	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2049	*/
2050	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2051	{
2052	PCIEMTB const pTb = pReNative->pTbOrg;
2053	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2054	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2055	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2056	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2057	uint32_t const idxRange2 = idxRange1 + 1;
2058	BODY_SET_CUR_INSTR();
2059	BODY_FLUSH_PENDING_WRITES();
2060	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2061	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2062	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2063	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2064	return off;
2065	}
2066
2067	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2068	{
2069	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2070	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2071	LIVENESS_CHECK_OPCODES(pOutgoing);
2072	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2073	RT_NOREF(pCallEntry);
2074	}
2075	#endif
2076
2077
2078	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2079	/**
2080	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2081	* advancing naturally to a different code page.
2082	*
2083	* Only opcodes on the new page is checked.
2084	*
2085	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb
2086	*/
2087	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2088	{
2089	PCIEMTB const pTb = pReNative->pTbOrg;
2090	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2091	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2092	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2093	//uint32_t const offRange1 = (uint32_t)uParam2;
2094	uint32_t const idxRange2 = idxRange1 + 1;
2095	BODY_SET_CUR_INSTR();
2096	BODY_FLUSH_PENDING_WRITES();
2097	BODY_CHECK_CS_LIM(cbInstr);
2098	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2099	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2100	return off;
2101	}
2102
2103	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2104	{
2105	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2106	LIVENESS_CHECK_CS_LIM(pOutgoing);
2107	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2108	LIVENESS_CHECK_OPCODES(pOutgoing);
2109	RT_NOREF(pCallEntry);
2110	}
2111	#endif
2112
2113
2114	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2115	/**
2116	* Built-in function for loading TLB and checking opcodes when advancing
2117	* naturally to a different code page.
2118	*
2119	* Only opcodes on the new page is checked.
2120	*
2121	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2122	*/
2123	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2124	{
2125	PCIEMTB const pTb = pReNative->pTbOrg;
2126	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2127	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2128	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2129	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2130	uint32_t const idxRange2 = idxRange1 + 1;
2131	BODY_SET_CUR_INSTR();
2132	BODY_FLUSH_PENDING_WRITES();
2133	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2134	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2135	return off;
2136	}
2137
2138	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2139	{
2140	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2141	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2142	LIVENESS_CHECK_OPCODES(pOutgoing);
2143	RT_NOREF(pCallEntry);
2144	}
2145	#endif
2146
2147
2148	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2149	/**
2150	* Built-in function for loading TLB and checking opcodes and considering the
2151	* need for CS.LIM checking when advancing naturally to a different code page.
2152	*
2153	* Only opcodes on the new page is checked.
2154	*
2155	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2156	*/
2157	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2158	{
2159	PCIEMTB const pTb = pReNative->pTbOrg;
2160	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2161	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2162	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2163	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2164	uint32_t const idxRange2 = idxRange1 + 1;
2165	BODY_SET_CUR_INSTR();
2166	BODY_FLUSH_PENDING_WRITES();
2167	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2168	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2169	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2170	return off;
2171	}
2172
2173	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2174	{
2175	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2176	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2177	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2178	LIVENESS_CHECK_OPCODES(pOutgoing);
2179	RT_NOREF(pCallEntry);
2180	}
2181	#endif
2182
2183
2184	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2185	/**
2186	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2187	* advancing naturally to a different code page with first instr at byte 0.
2188	*
2189	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb
2190	*/
2191	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2192	{
2193	PCIEMTB const pTb = pReNative->pTbOrg;
2194	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2195	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2196	BODY_SET_CUR_INSTR();
2197	BODY_FLUSH_PENDING_WRITES();
2198	BODY_CHECK_CS_LIM(cbInstr);
2199	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2200	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2201	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2202	return off;
2203	}
2204
2205	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2206	{
2207	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2208	LIVENESS_CHECK_CS_LIM(pOutgoing);
2209	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2210	LIVENESS_CHECK_OPCODES(pOutgoing);
2211	RT_NOREF(pCallEntry);
2212	}
2213	#endif
2214
2215
2216	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2217	/**
2218	* Built-in function for loading TLB and checking opcodes when advancing
2219	* naturally to a different code page with first instr at byte 0.
2220	*
2221	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2222	*/
2223	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2224	{
2225	PCIEMTB const pTb = pReNative->pTbOrg;
2226	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2227	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2228	BODY_SET_CUR_INSTR();
2229	BODY_FLUSH_PENDING_WRITES();
2230	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2231	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2232	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2233	return off;
2234	}
2235
2236	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2237	{
2238	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2239	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2240	LIVENESS_CHECK_OPCODES(pOutgoing);
2241	RT_NOREF(pCallEntry);
2242	}
2243	#endif
2244
2245
2246	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2247	/**
2248	* Built-in function for loading TLB and checking opcodes and considering the
2249	* need for CS.LIM checking when advancing naturally to a different code page
2250	* with first instr at byte 0.
2251	*
2252	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2253	*/
2254	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2255	{
2256	PCIEMTB const pTb = pReNative->pTbOrg;
2257	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2258	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2259	BODY_SET_CUR_INSTR();
2260	BODY_FLUSH_PENDING_WRITES();
2261	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2262	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2263	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2264	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2265	return off;
2266	}
2267
2268	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2269	{
2270	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2271	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2272	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2273	LIVENESS_CHECK_OPCODES(pOutgoing);
2274	RT_NOREF(pCallEntry);
2275	}
2276	#endif
2277

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp@ 103583

Download in other formats: