IEMAllN8veRecompBltIn.cpp

Last change on this file was 108369, checked in by vboxsync, 3 weeks ago
VMM/IEM: Renamed X86 specific IEMTB_F_XXX flags. jiraref:VBP-1531
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 105.6 KB

Line
1	/* $Id: IEMAllN8veRecompBltIn.cpp 108369 2025-02-25 13:23:41Z vboxsync $ */
2	/** @file
3	* IEM - Native Recompiler, Emitters for Built-In Threaded Functions.
4	*/
5
6	/*
7	* Copyright (C) 2023-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
33	#define IEM_WITH_OPAQUE_DECODER_STATE
34	#define VMCPU_INCL_CPUM_GST_CTX
35	#define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
36	#ifdef IN_RING0
37	# define VBOX_VMM_TARGET_X86
38	#endif
39	#include <VBox/vmm/iem.h>
40	#include <VBox/vmm/cpum.h>
41	#include <VBox/vmm/dbgf.h>
42	#include "IEMInternal.h"
43	#include <VBox/vmm/vmcc.h>
44	#include <VBox/log.h>
45	#include <VBox/err.h>
46	#include <VBox/param.h>
47	#include <iprt/assert.h>
48	#include <iprt/string.h>
49	#if defined(RT_ARCH_AMD64)
50	# include <iprt/x86.h>
51	#elif defined(RT_ARCH_ARM64)
52	# include <iprt/armv8.h>
53	#endif
54
55
56	#include "IEMInline.h"
57	#include "IEMThreadedFunctions.h"
58	#include "IEMN8veRecompiler.h"
59	#include "IEMN8veRecompilerEmit.h"
60	#include "IEMN8veRecompilerTlbLookup.h"
61	#include "target-x86/IEMAllN8veEmit-x86.h"
62
63
64
65	/*********************************************************************************************************************************
66	* TB Helper Functions *
67	*********************************************************************************************************************************/
68	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
69	DECLASM(void) iemNativeHlpAsmSafeWrapLogCpuState(void);
70	#endif
71
72
73	/**
74	* Used by TB code to deal with a TLB miss for a new page.
75	*/
76	IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemCodeNewPageTlbMiss,(PVMCPUCC pVCpu))
77	{
78	#ifdef IEM_WITH_TLB_STATISTICS
79	STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage);
80	#endif
81	pVCpu->iem.s.pbInstrBuf = NULL;
82	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE;
83	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
84	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
85	if (pVCpu->iem.s.pbInstrBuf)
86	{ /* likely */ }
87	else
88	{
89	AssertMsgFailed(("cs:rip=%04x:%08RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
90	IEM_DO_LONGJMP(pVCpu, VINF_SUCCESS);
91	}
92	}
93
94
95	/**
96	* Used by TB code to deal with a TLB miss for a new page.
97	*/
98	IEM_DECL_NATIVE_HLP_DEF(RTGCPHYS, iemNativeHlpMemCodeNewPageTlbMissWithOff,(PVMCPUCC pVCpu, uint8_t offInstr))
99	{
100	#ifdef IEM_WITH_TLB_STATISTICS
101	STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset);
102	#endif
103	pVCpu->iem.s.pbInstrBuf = NULL;
104	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE - offInstr;
105	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
106	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
107	AssertMsg(pVCpu->iem.s.pbInstrBuf, ("cs:rip=%04x:%08RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
108	return pVCpu->iem.s.pbInstrBuf ? pVCpu->iem.s.GCPhysInstrBuf : NIL_RTGCPHYS;
109	}
110
111
112	/*********************************************************************************************************************************
113	* Builtin functions *
114	*********************************************************************************************************************************/
115
116	/**
117	* Built-in function that does nothing.
118	*
119	* Whether this is called or not can be controlled by the entry in the
120	* IEMThreadedGenerator.katBltIns table. This can be useful to determine
121	* whether why behaviour changes when enabling the LogCpuState builtins. I.e.
122	* whether it's the reduced call count in the TBs or the threaded calls flushing
123	* register state.
124	*/
125	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Nop)
126	{
127	RT_NOREF(pReNative, pCallEntry);
128	return off;
129	}
130
131	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Nop)
132	{
133	pOutgoing = pIncoming;
134	RT_NOREF(pCallEntry);
135	}
136
137
138	/**
139	* Emits for for LogCpuState.
140	*
141	* This shouldn't have any relevant impact on the recompiler state.
142	*/
143	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_LogCpuState)
144	{
145	#ifdef RT_ARCH_AMD64
146	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 20);
147	/* push rax */
148	pbCodeBuf[off++] = 0x50 + X86_GREG_xAX;
149	/* push imm32 */
150	pbCodeBuf[off++] = 0x68;
151	pbCodeBuf[off++] = RT_BYTE1(pCallEntry->auParams[0]);
152	pbCodeBuf[off++] = RT_BYTE2(pCallEntry->auParams[0]);
153	pbCodeBuf[off++] = RT_BYTE3(pCallEntry->auParams[0]);
154	pbCodeBuf[off++] = RT_BYTE4(pCallEntry->auParams[0]);
155	/* mov rax, iemNativeHlpAsmSafeWrapLogCpuState */
156	pbCodeBuf[off++] = X86_OP_REX_W;
157	pbCodeBuf[off++] = 0xb8 + X86_GREG_xAX;
158	(uint64_t )&pbCodeBuf[off] = (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState;
159	off += sizeof(uint64_t);
160	/* call rax */
161	pbCodeBuf[off++] = 0xff;
162	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
163	/* pop rax */
164	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
165	/* pop rax */
166	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
167	#else
168	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState);
169	RT_NOREF(pCallEntry);
170	#endif
171
172	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
173	return off;
174	}
175
176	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_LogCpuState)
177	{
178	IEM_LIVENESS_RAW_INIT_WITH_CALL(pOutgoing, pIncoming);
179	RT_NOREF(pCallEntry);
180	}
181
182
183	/**
184	* Built-in function that calls a C-implemention function taking zero arguments.
185	*/
186	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_DeferToCImpl0)
187	{
188	PFNIEMCIMPL0 const pfnCImpl = (PFNIEMCIMPL0)(uintptr_t)pCallEntry->auParams[0];
189	uint8_t const cbInstr = (uint8_t)pCallEntry->auParams[1];
190	uint64_t const fGstShwFlush = pCallEntry->auParams[2];
191	return iemNativeEmitCImplCall(pReNative, off, pCallEntry->idxInstr, fGstShwFlush, (uintptr_t)pfnCImpl, cbInstr, 0, 0, 0, 0);
192	}
193
194	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_DeferToCImpl0)
195	{
196	IEM_LIVENESS_RAW_INIT_WITH_CALL(pOutgoing, pIncoming);
197	RT_NOREF(pCallEntry);
198	}
199
200
201	/**
202	* Flushes pending writes in preparation of raising an exception or aborting the TB.
203	*/
204	#define BODY_FLUSH_PENDING_WRITES() \
205	off = iemNativeRegFlushPendingWrites(pReNative, off);
206
207
208	/**
209	* Worker for the CheckIrq, CheckTimers and CheckTimersAndIrq builtins below.
210	*/
211	template<bool const a_fCheckTimers, bool const a_fCheckIrqs>
212	DECL_FORCE_INLINE_THROW(uint32_t) iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off)
213	{
214	uint8_t const idxEflReg = !a_fCheckIrqs ? UINT8_MAX
215	: iemNativeRegAllocTmpForGuestEFlagsReadOnly(pReNative, &off,
216	RT_BIT_64(IEMLIVENESSBIT_IDX_EFL_OTHER));
217	uint8_t const idxTmpReg1 = iemNativeRegAllocTmp(pReNative, &off);
218	uint8_t const idxTmpReg2 = a_fCheckIrqs ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
219	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off,
220	(RT_ARCH_VAL == RT_ARCH_VAL_AMD64 ? 72 : 32)
221	+ IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS * 3);
222
223	/*
224	* First we decrement the timer poll counter, if so desired.
225	*/
226	if (a_fCheckTimers)
227	{
228	# ifdef RT_ARCH_AMD64
229	/* dec [rbx + cTbsTillNextTimerPoll] */
230	pCodeBuf[off++] = 0xff;
231	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, 1, RT_UOFFSETOF(VMCPU, iem.s.cTbsTillNextTimerPoll));
232
233	/* jz ReturnBreakFF */
234	off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_ReturnBreakFF>(pReNative, pCodeBuf, off, kIemNativeInstrCond_e);
235
236	# elif defined(RT_ARCH_ARM64)
237	AssertCompile(RTASSERT_OFFSET_OF(VMCPU, iem.s.cTbsTillNextTimerPoll) < _4K * sizeof(uint32_t));
238	off = iemNativeEmitLoadGprFromVCpuU32Ex(pCodeBuf, off, idxTmpReg1, RT_UOFFSETOF(VMCPU, iem.s.cTbsTillNextTimerPoll));
239	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxTmpReg1, idxTmpReg1, 1, false /f64Bit/);
240	off = iemNativeEmitStoreGprToVCpuU32Ex(pCodeBuf, off, idxTmpReg1, RT_UOFFSETOF(VMCPU, iem.s.cTbsTillNextTimerPoll));
241
242	/* cbz reg1, ReturnBreakFF */
243	off = iemNativeEmitTbExitIfGprIsZeroEx<kIemNativeLabelType_ReturnBreakFF>(pReNative, pCodeBuf, off,
244	idxTmpReg1, false /f64Bit/);
245
246	# else
247	# error "port me"
248	# endif
249	}
250
251	/*
252	* Second, check forced flags, if so desired.
253	*
254	* We OR them together to save a conditional. A trick here is that the
255	* two IRQ flags are unused in the global flags, so we can still use the
256	* resulting value to check for suppressed interrupts.
257	*/
258	if (a_fCheckIrqs)
259	{
260	/* Load VMCPU::fLocalForcedActions first and mask it. We can simplify the
261	masking by ASSUMING none of the unwanted flags are located above bit 30. */
262	uint64_t const fUnwantedCpuFFs = VMCPU_FF_PGM_SYNC_CR3
263	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
264	\| VMCPU_FF_TLB_FLUSH
265	\| VMCPU_FF_UNHALT;
266	AssertCompile(fUnwantedCpuFFs < RT_BIT_64(31));
267	off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, idxTmpReg1, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
268	# if defined(RT_ARCH_AMD64)
269	/* and reg1, ~fUnwantedCpuFFs */
270	pCodeBuf[off++] = idxTmpReg1 >= 8 ? X86_OP_REX_B \| X86_OP_REX_W : X86_OP_REX_W;
271	pCodeBuf[off++] = 0x81;
272	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, idxTmpReg1 & 7);
273	(uint32_t )&pCodeBuf[off] = ~(uint32_t)fUnwantedCpuFFs;
274	off += 4;
275
276	# else
277	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxTmpReg2, ~fUnwantedCpuFFs);
278	off = iemNativeEmitAndGprByGprEx(pCodeBuf, off, idxTmpReg1, idxTmpReg2);
279	# endif
280
281	/* OR in VM::fGlobalForcedActions. We access the member via pVCpu.
282	No need to mask anything here. Unfortunately, it's a 32-bit
283	variable, so we can't OR it directly on x86.
284
285	We try avoid loading the pVM address here by seeing if we can access
286	VM::fGlobalForcedActions via 32-bit pVCpu displacement. For
287	driverless configs, this should go without problems (negative
288	offset), but when using the driver the layout is randomized and we
289	need to be flexible. */
290	AssertCompile(VM_FF_ALL_MASK == UINT32_MAX);
291	uintptr_t const uAddrGlobalForcedActions = (uintptr_t)&pReNative->pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions;
292	intptr_t const offGlobalForcedActions = (intptr_t)uAddrGlobalForcedActions - (intptr_t)pReNative->pVCpu;
293
294	# ifdef RT_ARCH_AMD64
295	if (RT_LIKELY((int32_t)offGlobalForcedActions == offGlobalForcedActions))
296	{
297	if (idxTmpReg2 >= 8)
298	pCodeBuf[off++] = X86_OP_REX_R;
299	pCodeBuf[off++] = 0x8b; /* mov */
300	off = iemNativeEmitGprByVCpuSignedDisp(pCodeBuf, off, idxTmpReg2, (int32_t)offGlobalForcedActions);
301	}
302	else
303	{
304	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxTmpReg2, uAddrGlobalForcedActions);
305
306	if (idxTmpReg2 >= 8)
307	pCodeBuf[off++] = X86_OP_REX_R \| X86_OP_REX_B;
308	pCodeBuf[off++] = 0x8b; /* mov */
309	iemNativeEmitGprByGprDisp(pCodeBuf, off, idxTmpReg2, idxTmpReg2, 0);
310	}
311
312	/* or reg1, reg2 */
313	off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, idxTmpReg1, idxTmpReg2);
314
315	/* jz nothing_pending */
316	uint32_t const offFixup1 = off;
317	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, IEMNATIVE_HAS_POSTPONED_EFLAGS_CALCS(pReNative) ? off + 512 : off + 64,
318	kIemNativeInstrCond_e);
319
320	# elif defined(RT_ARCH_ARM64)
321	if (RT_LIKELY((int32_t)offGlobalForcedActions == offGlobalForcedActions))
322	{
323	if (offGlobalForcedActions < 0)
324	off = iemNativeEmitGprBySignedVCpuLdStEx(pCodeBuf, off, idxTmpReg2, (int32_t)offGlobalForcedActions,
325	kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
326	else
327	off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, idxTmpReg2, (uint32_t)offGlobalForcedActions,
328	kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
329	}
330	else
331	{
332	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxTmpReg2, uAddrGlobalForcedActions);
333	pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxTmpReg2, idxTmpReg2, 0);
334	}
335
336	off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, idxTmpReg1, idxTmpReg2);
337
338	/* cbz nothing_pending */
339	uint32_t const offFixup1 = off;
340	off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToFixedEx(pCodeBuf, off, idxTmpReg1, true /f64Bit/,
341	false /fJmpIfNotZero/, off);
342	# else
343	# error "port me"
344	# endif
345
346	/* More than just IRQ FFs pending? */
347	AssertCompile((VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC) == 3);
348	/* cmp reg1, 3 */
349	off = iemNativeEmitCmpGprWithImmEx(pCodeBuf, off, idxTmpReg1, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC);
350	/* ja ReturnBreakFF */
351	off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_ReturnBreakFF>(pReNative, pCodeBuf, off, kIemNativeInstrCond_nbe);
352
353	/*
354	* Okay, we've only got pending IRQ related FFs: Can we dispatch IRQs?
355	*
356	* ASSUME that the shadow flags are cleared when they ought to be cleared,
357	* so we can skip the RIP check.
358	*/
359	AssertCompile(CPUMCTX_INHIBIT_SHADOW < RT_BIT_32(31));
360	/* reg1 = efl & (IF \| INHIBIT_SHADOW) */
361	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxTmpReg1, idxEflReg, X86_EFL_IF \| CPUMCTX_INHIBIT_SHADOW);
362	/* reg1 ^= IF */
363	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, idxTmpReg1, X86_EFL_IF);
364
365	# ifdef RT_ARCH_AMD64
366	/* jz ReturnBreakFF */
367	off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_ReturnBreakFF>(pReNative, pCodeBuf, off, kIemNativeInstrCond_e);
368
369	# elif defined(RT_ARCH_ARM64)
370	/* cbz reg1, ReturnBreakFF */
371	off = iemNativeEmitTbExitIfGprIsZeroEx<kIemNativeLabelType_ReturnBreakFF>(pReNative, pCodeBuf, off,
372	idxTmpReg1, false /f64Bit/);
373	# else
374	# error "port me"
375	# endif
376	/*
377	* nothing_pending:
378	*/
379	iemNativeFixupFixedJump(pReNative, offFixup1, off);
380	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
381	}
382
383	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
384
385	/*
386	* Cleanup.
387	*/
388	iemNativeRegFreeTmp(pReNative, idxTmpReg1);
389	if (a_fCheckIrqs)
390	{
391	iemNativeRegFreeTmp(pReNative, idxTmpReg2);
392	iemNativeRegFreeTmp(pReNative, idxEflReg);
393	}
394	else
395	{
396	Assert(idxTmpReg2 == UINT8_MAX);
397	Assert(idxEflReg == UINT8_MAX);
398	}
399
400	return off;
401	}
402
403
404	/**
405	* Built-in function that checks for pending interrupts that can be delivered or
406	* forced action flags.
407	*
408	* This triggers after the completion of an instruction, so EIP is already at
409	* the next instruction. If an IRQ or important FF is pending, this will return
410	* a non-zero status that stops TB execution.
411	*/
412	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
413	{
414	BODY_FLUSH_PENDING_WRITES();
415	off = iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon<false, true>(pReNative, off);
416
417	/* Note down that we've been here, so we can skip FFs + IRQ checks when
418	doing direct linking. */
419	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
420	pReNative->idxLastCheckIrqCallNo = pReNative->idxCurCall;
421	RT_NOREF(pCallEntry);
422	#else
423	pReNative->idxLastCheckIrqCallNo = pCallEntry - pReNative->pTbOrg->Thrd.paCalls;
424	#endif
425
426	return off;
427	}
428
429	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckIrq)
430	{
431	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
432	IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
433	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
434	RT_NOREF(pCallEntry);
435	}
436
437
438	/**
439	* Built-in function that works the cTbsTillNextTimerPoll counter on direct TB
440	* linking, like loop-jumps.
441	*/
442	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckTimers)
443	{
444	BODY_FLUSH_PENDING_WRITES();
445	RT_NOREF(pCallEntry);
446	return iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon<true, false>(pReNative, off);
447	}
448
449	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckTimers)
450	{
451	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
452	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
453	RT_NOREF(pCallEntry);
454	}
455
456
457	/**
458	* Combined BltIn_CheckTimers + BltIn_CheckIrq for direct linking.
459	*/
460	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckTimersAndIrq)
461	{
462	BODY_FLUSH_PENDING_WRITES();
463	RT_NOREF(pCallEntry);
464	return iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon<true, true>(pReNative, off);
465	}
466
467	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckTimersAndIrq)
468	{
469	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
470	IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
471	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
472	RT_NOREF(pCallEntry);
473	}
474
475
476	/**
477	* Built-in function checks if IEMCPU::fExec has the expected value.
478	*/
479	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckMode)
480	{
481	uint32_t const fExpectedExec = (uint32_t)pCallEntry->auParams[0];
482	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
483
484	off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
485	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
486	off = iemNativeEmitTbExitIfGpr32NotEqualImm<kIemNativeLabelType_ReturnBreak>(pReNative, off, idxTmpReg,
487	fExpectedExec & IEMTB_F_KEY_MASK);
488	iemNativeRegFreeTmp(pReNative, idxTmpReg);
489
490	/* Maintain the recompiler fExec state. */
491	pReNative->fExec = fExpectedExec & IEMTB_F_IEM_F_MASK;
492	return off;
493	}
494
495	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckMode)
496	{
497	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
498	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
499	RT_NOREF(pCallEntry);
500	}
501
502
503	/**
504	* Sets idxTbCurInstr in preparation of raising an exception or aborting the TB.
505	*/
506	/** @todo Optimize this, so we don't set the same value more than once. Just
507	* needs some tracking. */
508	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
509	# define BODY_SET_CUR_INSTR() \
510	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, pCallEntry->idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr))
511	#else
512	# define BODY_SET_CUR_INSTR() ((void)0)
513	#endif
514
515
516	/**
517	* Macro that emits the 16/32-bit CS.LIM check.
518	*/
519	#define BODY_CHECK_CS_LIM(a_cbInstr) \
520	off = iemNativeEmitBltInCheckCsLim(pReNative, off, (a_cbInstr))
521
522	#define LIVENESS_CHECK_CS_LIM(a_pOutgoing) \
523	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS)
524
525	DECL_FORCE_INLINE(uint32_t)
526	iemNativeEmitBltInCheckCsLim(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
527	{
528	Assert(cbInstr > 0);
529	Assert(cbInstr < 16);
530	#ifdef VBOX_STRICT
531	off = iemNativeEmitMarker(pReNative, off, 0x80000001);
532	#endif
533
534	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
535	Assert(pReNative->Core.offPc == 0);
536	#endif
537
538	/*
539	* We need CS.LIM and RIP here. When cbInstr is larger than 1, we also need
540	* a temporary register for calculating the last address of the instruction.
541	*
542	* The calculation and comparisons are 32-bit. We ASSUME that the incoming
543	* RIP isn't totally invalid, i.e. that any jump/call/ret/iret instruction
544	* that last updated EIP here checked it already, and that we're therefore
545	* safe in the 32-bit wrap-around scenario to only check that the last byte
546	* is within CS.LIM. In the case of instruction-by-instruction advancing
547	* up to a EIP wrap-around, we know that CS.LIM is 4G-1 because the limit
548	* must be using 4KB granularity and the previous instruction was fine.
549	*/
550	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
551	kIemNativeGstRegUse_ReadOnly);
552	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
553	kIemNativeGstRegUse_ReadOnly);
554	#ifdef RT_ARCH_AMD64
555	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
556	#elif defined(RT_ARCH_ARM64)
557	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
558	#else
559	# error "Port me"
560	#endif
561
562	if (cbInstr != 1)
563	{
564	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
565
566	/*
567	* 1. idxRegTmp = idxRegPc + cbInstr;
568	* 2. if idxRegTmp > idxRegCsLim then raise #GP(0).
569	*/
570	#ifdef RT_ARCH_AMD64
571	/* 1. lea tmp32, [Pc + cbInstr - 1] */
572	if (idxRegTmp >= 8 \|\| idxRegPc >= 8)
573	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegPc < 8 ? 0 : X86_OP_REX_B);
574	pbCodeBuf[off++] = 0x8d;
575	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, idxRegTmp & 7, idxRegPc & 7);
576	if ((idxRegPc & 7) == X86_GREG_xSP)
577	pbCodeBuf[off++] = X86_SIB_MAKE(idxRegPc & 7, 4 /no index/, 0);
578	pbCodeBuf[off++] = cbInstr - 1;
579
580	/* 2. cmp tmp32(r), CsLim(r/m). */
581	if (idxRegTmp >= 8 \|\| idxRegCsLim >= 8)
582	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
583	pbCodeBuf[off++] = 0x3b;
584	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegTmp & 7, idxRegCsLim & 7);
585
586	#elif defined(RT_ARCH_ARM64)
587	/* 1. add tmp32, Pc, #cbInstr-1 */
588	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxRegTmp, idxRegPc, cbInstr - 1, false /f64Bit/);
589	/* 2. cmp tmp32, CsLim */
590	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegTmp, idxRegCsLim,
591	false /f64Bit/, true /fSetFlags/);
592
593	#endif
594	iemNativeRegFreeTmp(pReNative, idxRegTmp);
595	}
596	else
597	{
598	/*
599	* Here we can skip step 1 and compare PC and CS.LIM directly.
600	*/
601	#ifdef RT_ARCH_AMD64
602	/* 2. cmp eip(r), CsLim(r/m). */
603	if (idxRegPc >= 8 \|\| idxRegCsLim >= 8)
604	pbCodeBuf[off++] = (idxRegPc < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
605	pbCodeBuf[off++] = 0x3b;
606	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegPc & 7, idxRegCsLim & 7);
607
608	#elif defined(RT_ARCH_ARM64)
609	/* 2. cmp Pc, CsLim */
610	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegPc, idxRegCsLim,
611	false /f64Bit/, true /fSetFlags/);
612
613	#endif
614	}
615
616	/* 3. Jump if greater. */
617	off = iemNativeEmitTbExitJa<kIemNativeLabelType_RaiseGp0>(pReNative, off);
618
619	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
620	iemNativeRegFreeTmp(pReNative, idxRegPc);
621	return off;
622	}
623
624
625	/**
626	* Macro that considers whether we need CS.LIM checking after a branch or
627	* crossing over to a new page.
628	*/
629	#define BODY_CONSIDER_CS_LIM_CHECKING(a_pTb, a_cbInstr) \
630	RT_NOREF(a_cbInstr); \
631	off = iemNativeEmitBltInConsiderLimChecking(pReNative, off)
632
633	#define LIVENESS_CONSIDER_CS_LIM_CHECKING(a_pOutgoing) \
634	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS); \
635	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS)
636
637	DECL_FORCE_INLINE(uint32_t)
638	iemNativeEmitBltInConsiderLimChecking(PIEMRECOMPILERSTATE pReNative, uint32_t off)
639	{
640	#ifdef VBOX_STRICT
641	off = iemNativeEmitMarker(pReNative, off, 0x80000002);
642	#endif
643
644	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
645	Assert(pReNative->Core.offPc == 0);
646	#endif
647
648	/*
649	* This check must match the ones in the iem in iemGetTbFlagsForCurrentPc
650	* exactly:
651	*
652	* int64_t const offFromLim = (int64_t)pVCpu->cpum.GstCtx.cs.u32Limit - (int64_t)pVCpu->cpum.GstCtx.eip;
653	* if (offFromLim >= X86_PAGE_SIZE + 16 - (int32_t)(pVCpu->cpum.GstCtx.cs.u64Base & GUEST_PAGE_OFFSET_MASK))
654	* return fRet;
655	* return fRet \| IEMTB_F_X86_CS_LIM_CHECKS;
656	*
657	*
658	* We need EIP, CS.LIM and CS.BASE here.
659	*/
660
661	/* Calculate the offFromLim first: */
662	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
663	kIemNativeGstRegUse_ReadOnly);
664	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
665	kIemNativeGstRegUse_ReadOnly);
666	uint8_t const idxRegLeft = iemNativeRegAllocTmp(pReNative, &off);
667
668	#ifdef RT_ARCH_ARM64
669	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
670	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegLeft, idxRegCsLim, idxRegPc);
671	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
672	#else
673	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegLeft, idxRegCsLim);
674	off = iemNativeEmitSubTwoGprs(pReNative, off, idxRegLeft, idxRegPc);
675	#endif
676
677	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
678	iemNativeRegFreeTmp(pReNative, idxRegPc);
679
680	/* Calculate the threshold level (right side). */
681	uint8_t const idxRegCsBase = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
682	kIemNativeGstRegUse_ReadOnly);
683	uint8_t const idxRegRight = iemNativeRegAllocTmp(pReNative, &off);
684
685	#ifdef RT_ARCH_ARM64
686	pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
687	Assert(Armv8A64ConvertImmRImmS2Mask32(11, 0) == GUEST_PAGE_OFFSET_MASK);
688	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegRight, idxRegCsBase, 11, 0, false /f64Bit/);
689	pu32CodeBuf[off++] = Armv8A64MkInstrNeg(idxRegRight);
690	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
691	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
692	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
693
694	#else
695	off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegRight, GUEST_PAGE_OFFSET_MASK);
696	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegRight, idxRegCsBase);
697	off = iemNativeEmitNegGpr(pReNative, off, idxRegRight);
698	off = iemNativeEmitAddGprImm(pReNative, off, idxRegRight, X86_PAGE_SIZE + 16);
699	#endif
700
701	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
702
703	/* Compare the two and jump out if we're too close to the limit. */
704	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
705	off = iemNativeEmitTbExitJl<kIemNativeLabelType_NeedCsLimChecking>(pReNative, off);
706
707	iemNativeRegFreeTmp(pReNative, idxRegRight);
708	iemNativeRegFreeTmp(pReNative, idxRegLeft);
709	return off;
710	}
711
712
713
714	/**
715	* Macro that implements opcode (re-)checking.
716	*/
717	#define BODY_CHECK_OPCODES(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
718	RT_NOREF(a_cbInstr); \
719	off = iemNativeEmitBltInCheckOpcodes(pReNative, off, (a_pTb), (a_idxRange), (a_offRange))
720
721	#define LIVENESS_CHECK_OPCODES(a_pOutgoing) ((void)0)
722
723	#if 0 /* debugging aid */
724	bool g_fBpOnObsoletion = false;
725	# define BP_ON_OBSOLETION g_fBpOnObsoletion
726	#else
727	# define BP_ON_OBSOLETION 0
728	#endif
729
730	DECL_FORCE_INLINE(uint32_t)
731	iemNativeEmitBltInCheckOpcodes(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint16_t offRange)
732	{
733	Assert(idxRange < pTb->cRanges && pTb->cRanges <= RT_ELEMENTS(pTb->aRanges));
734	Assert(offRange < pTb->aRanges[idxRange].cbOpcodes);
735	#ifdef VBOX_STRICT
736	off = iemNativeEmitMarker(pReNative, off, 0x80000003);
737	#endif
738
739	/*
740	* Where to start and how much to compare.
741	*
742	* Looking at the ranges produced when r160746 was running a DOS VM with TB
743	* logging, the ranges can be anything from 1 byte to at least 0x197 bytes,
744	* with the 6, 5, 4, 7, 8, 40, 3, 2, 9 and 10 being the top 10 in the sample.
745	*
746	* The top 10 for the early boot phase of a 64-bit debian 9.4 VM: 5, 9, 8,
747	* 12, 10, 11, 6, 13, 15 and 16. Max 0x359 bytes. Same revision as above.
748	*/
749	uint16_t offPage = pTb->aRanges[idxRange].offPhysPage + offRange;
750	uint16_t cbLeft = pTb->aRanges[idxRange].cbOpcodes - offRange;
751	Assert(cbLeft > 0);
752	uint8_t const *pbOpcodes = &pTb->pabOpcodes[pTb->aRanges[idxRange].offOpcodes + offRange];
753	uint32_t offConsolidatedJump = UINT32_MAX;
754
755	#ifdef RT_ARCH_AMD64
756	/* AMD64/x86 offers a bunch of options. Smaller stuff will can be
757	completely inlined, for larger we use REPE CMPS. */
758	# define CHECK_OPCODES_CMP_IMMXX(a_idxReg, a_bOpcode) /* cost: 3 bytes */ do { \
759	pbCodeBuf[off++] = a_bOpcode; \
760	Assert(offPage < 127); \
761	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 7, a_idxReg); \
762	pbCodeBuf[off++] = RT_BYTE1(offPage); \
763	} while (0)
764
765	# define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
766	if (offConsolidatedJump != UINT32_MAX) \
767	{ \
768	int32_t const offDisp = (int32_t)offConsolidatedJump - (int32_t)(off + 2); \
769	Assert(offDisp >= -128); \
770	pbCodeBuf[off++] = 0x75; /* jnz near */ \
771	pbCodeBuf[off++] = (uint8_t)offDisp; \
772	} \
773	else \
774	{ \
775	pbCodeBuf[off++] = 0x74; /* jz near +5 */ \
776	offConsolidatedJump = ++off; \
777	if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
778	off = iemNativeEmitTbExitEx<kIemNativeLabelType_ObsoleteTb, false /a_fActuallyExitingTb/>(pReNative, \
779	pbCodeBuf, off); \
780	pbCodeBuf[offConsolidatedJump - 1] = off - offConsolidatedJump; \
781	} \
782	} while (0)
783
784	# define CHECK_OPCODES_CMP_IMM32(a_idxReg) /* cost: 3+4+2 = 9 */ do { \
785	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
786	pbCodeBuf[off++] = *pbOpcodes++; \
787	pbCodeBuf[off++] = *pbOpcodes++; \
788	pbCodeBuf[off++] = *pbOpcodes++; \
789	pbCodeBuf[off++] = *pbOpcodes++; \
790	cbLeft -= 4; \
791	offPage += 4; \
792	CHECK_OPCODES_CMP_JMP(); \
793	} while (0)
794
795	# define CHECK_OPCODES_CMP_IMM16(a_idxReg) /* cost: 1+3+2+2 = 8 */ do { \
796	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP; \
797	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
798	pbCodeBuf[off++] = *pbOpcodes++; \
799	pbCodeBuf[off++] = *pbOpcodes++; \
800	cbLeft -= 2; \
801	offPage += 2; \
802	CHECK_OPCODES_CMP_JMP(); \
803	} while (0)
804
805	# define CHECK_OPCODES_CMP_IMM8(a_idxReg) /* cost: 3+1+2 = 6 */ do { \
806	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x80); \
807	pbCodeBuf[off++] = *pbOpcodes++; \
808	cbLeft -= 1; \
809	offPage += 1; \
810	CHECK_OPCODES_CMP_JMP(); \
811	} while (0)
812
813	# define CHECK_OPCODES_CMPSX(a_bOpcode, a_cbToSubtract, a_bPrefix) /* cost: 2+2 = 4 */ do { \
814	if (a_bPrefix) \
815	pbCodeBuf[off++] = (a_bPrefix); \
816	pbCodeBuf[off++] = (a_bOpcode); \
817	CHECK_OPCODES_CMP_JMP(); \
818	cbLeft -= (a_cbToSubtract); \
819	} while (0)
820
821	# define CHECK_OPCODES_ECX_IMM(a_uValue) /* cost: 5 */ do { \
822	pbCodeBuf[off++] = 0xb8 + X86_GREG_xCX; \
823	pbCodeBuf[off++] = RT_BYTE1(a_uValue); \
824	pbCodeBuf[off++] = RT_BYTE2(a_uValue); \
825	pbCodeBuf[off++] = RT_BYTE3(a_uValue); \
826	pbCodeBuf[off++] = RT_BYTE4(a_uValue); \
827	} while (0)
828
829	if (cbLeft <= 24)
830	{
831	uint8_t const idxRegTmp = iemNativeRegAllocTmpEx(pReNative, &off,
832	( RT_BIT_32(X86_GREG_xAX)
833	\| RT_BIT_32(X86_GREG_xCX)
834	\| RT_BIT_32(X86_GREG_xDX)
835	\| RT_BIT_32(X86_GREG_xBX)
836	\| RT_BIT_32(X86_GREG_xSI)
837	\| RT_BIT_32(X86_GREG_xDI))
838	& ~IEMNATIVE_REG_FIXED_MASK); /* pick reg not requiring rex prefix */
839	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf));
840	if (offPage >= 128 - cbLeft)
841	{
842	off = iemNativeEmitAddGprImm(pReNative, off, idxRegTmp, offPage & ~(uint16_t)3);
843	offPage &= 3;
844	}
845
846	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 88 */
847	+ IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
848
849	if (cbLeft > 8)
850	switch (offPage & 3)
851	{
852	case 0:
853	break;
854	case 1: /* cost: 6 + 8 = 14 */
855	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
856	RT_FALL_THRU();
857	case 2: /* cost: 8 */
858	CHECK_OPCODES_CMP_IMM16(idxRegTmp);
859	break;
860	case 3: /* cost: 6 */
861	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
862	break;
863	}
864
865	while (cbLeft >= 4)
866	CHECK_OPCODES_CMP_IMM32(idxRegTmp); /* max iteration: 24/4 = 6; --> cost: 6 * 9 = 54 */
867
868	if (cbLeft >= 2)
869	CHECK_OPCODES_CMP_IMM16(idxRegTmp); /* cost: 8 */
870	if (cbLeft)
871	CHECK_OPCODES_CMP_IMM8(idxRegTmp); /* cost: 6 */
872
873	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
874	iemNativeRegFreeTmp(pReNative, idxRegTmp);
875	}
876	else
877	{
878	/* RDI = &pbInstrBuf[offPage] */
879	uint8_t const idxRegDi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xDI));
880	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegDi, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
881	if (offPage != 0)
882	off = iemNativeEmitAddGprImm(pReNative, off, idxRegDi, offPage);
883
884	/* RSI = pbOpcodes */
885	uint8_t const idxRegSi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xSI));
886	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegSi, (uintptr_t)pbOpcodes);
887
888	/* RCX = counts. */
889	uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
890
891	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /= 36/
892	+ IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
893
894	/** @todo profile and optimize this further. Maybe an idea to align by
895	* offPage if the two cannot be reconsidled. */
896	/* Align by the page offset, so that at least one of the accesses are naturally aligned. */
897	switch (offPage & 7) /* max cost: 10 */
898	{
899	case 0:
900	break;
901	case 1: /* cost: 3+4+3 = 10 */
902	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
903	RT_FALL_THRU();
904	case 2: /* cost: 4+3 = 7 */
905	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
906	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
907	break;
908	case 3: /* cost: 3+3 = 6 */
909	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
910	RT_FALL_THRU();
911	case 4: /* cost: 3 */
912	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
913	break;
914	case 5: /* cost: 3+4 = 7 */
915	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
916	RT_FALL_THRU();
917	case 6: /* cost: 4 */
918	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
919	break;
920	case 7: /* cost: 3 */
921	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
922	break;
923	}
924
925	/* Compare qwords: */
926	uint32_t const cQWords = cbLeft >> 3;
927	CHECK_OPCODES_ECX_IMM(cQWords); /* cost: 5 */
928
929	pbCodeBuf[off++] = X86_OP_PRF_REPZ; /* cost: 5 */
930	CHECK_OPCODES_CMPSX(0xa7, 0, X86_OP_REX_W);
931	cbLeft &= 7;
932
933	if (cbLeft & 4)
934	CHECK_OPCODES_CMPSX(0xa7, 4, 0); /* cost: 3 */
935	if (cbLeft & 2)
936	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP); /* cost: 4 */
937	if (cbLeft & 1)
938	CHECK_OPCODES_CMPSX(0xa6, 1, 0); /* cost: 3 */
939
940	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
941	iemNativeRegFreeTmp(pReNative, idxRegCx);
942	iemNativeRegFreeTmp(pReNative, idxRegSi);
943	iemNativeRegFreeTmp(pReNative, idxRegDi);
944	}
945
946	#elif defined(RT_ARCH_ARM64)
947	/* We need pbInstrBuf in a register, whatever we do. */
948	uint8_t const idxRegSrc1Ptr = iemNativeRegAllocTmp(pReNative, &off);
949	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegSrc1Ptr, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
950
951	/* We also need at least one more register for holding bytes & words we
952	load via pbInstrBuf. */
953	uint8_t const idxRegSrc1Val = iemNativeRegAllocTmp(pReNative, &off);
954
955	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS * 2);
956
957	/* One byte compare can be done with the opcode byte as an immediate. We'll
958	do this to uint16_t align src1. */
959	bool fPendingJmp = RT_BOOL(offPage & 1);
960	if (fPendingJmp)
961	{
962	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
963	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes++, false /f64Bit*/);
964	offPage += 1;
965	cbLeft -= 1;
966	}
967
968	if (cbLeft > 0)
969	{
970	/* We need a register for holding the opcode bytes we're comparing with,
971	as CCMP only has a 5-bit immediate form and thus cannot hold bytes. */
972	uint8_t const idxRegSrc2Val = iemNativeRegAllocTmp(pReNative, &off);
973
974	/* Word (uint32_t) aligning the src1 pointer is best done using a 16-bit constant load. */
975	if ((offPage & 3) && cbLeft >= 2)
976	{
977	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 2);
978	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
979	if (fPendingJmp)
980	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
981	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
982	else
983	{
984	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
985	fPendingJmp = true;
986	}
987	pbOpcodes += 2;
988	offPage += 2;
989	cbLeft -= 2;
990	}
991
992	/* DWord (uint64_t) aligning the src2 pointer. We use a 32-bit constant here for simplicitly. */
993	if ((offPage & 7) && cbLeft >= 4)
994	{
995	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 4);
996	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
997	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
998	if (fPendingJmp)
999	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1000	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1001	else
1002	{
1003	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
1004	fPendingJmp = true;
1005	}
1006	pbOpcodes += 4;
1007	offPage += 4;
1008	cbLeft -= 4;
1009	}
1010
1011	/*
1012	* If we've got 16 bytes or more left, switch to memcmp-style.
1013	*/
1014	if (cbLeft >= 16)
1015	{
1016	/* We need a pointer to the copy of the original opcode bytes. */
1017	uint8_t const idxRegSrc2Ptr = iemNativeRegAllocTmp(pReNative, &off);
1018	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Ptr, (uintptr_t)pbOpcodes);
1019
1020	/* If there are more than 32 bytes to compare we create a loop, for
1021	which we'll need a loop register. */
1022	if (cbLeft >= 64)
1023	{
1024	if (fPendingJmp)
1025	{
1026	off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_ObsoleteTb>(pReNative, pu32CodeBuf, off,
1027	kArmv8InstrCond_Ne);
1028	fPendingJmp = false;
1029	}
1030
1031	uint8_t const idxRegLoop = iemNativeRegAllocTmp(pReNative, &off);
1032	uint16_t const cLoops = cbLeft / 32;
1033	cbLeft = cbLeft % 32;
1034	pbOpcodes += cLoops * 32;
1035	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegLoop, cLoops);
1036
1037	if (offPage != 0) /** @todo optimize out this instruction. */
1038	{
1039	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, offPage);
1040	offPage = 0;
1041	}
1042
1043	uint32_t const offLoopStart = off;
1044	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 0);
1045	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 0);
1046	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
1047
1048	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 1);
1049	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 1);
1050	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1051	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
1052
1053	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 2);
1054	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 2);
1055	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1056	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
1057
1058	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 3);
1059	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 3);
1060	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1061	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
1062
1063	off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_ObsoleteTb>(pReNative, pu32CodeBuf, off, kArmv8InstrCond_Ne);
1064
1065	/* Advance and loop. */
1066	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, 0x20);
1067	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc2Ptr, idxRegSrc2Ptr, 0x20);
1068	pu32CodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxRegLoop, idxRegLoop, 1, false /f64Bit/, true /fSetFlags/);
1069	pu32CodeBuf[off] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, (int32_t)offLoopStart - (int32_t)off);
1070	off++;
1071
1072	iemNativeRegFreeTmp(pReNative, idxRegLoop);
1073	}
1074
1075	/* Deal with any remaining dwords (uint64_t). There can be up to
1076	three if we looped and four if we didn't. */
1077	uint32_t offSrc2 = 0;
1078	while (cbLeft >= 8)
1079	{
1080	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val,
1081	idxRegSrc1Ptr, offPage / 8);
1082	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val,
1083	idxRegSrc2Ptr, offSrc2 / 8);
1084	if (fPendingJmp)
1085	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1086	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
1087	else
1088	{
1089	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
1090	fPendingJmp = true;
1091	}
1092	pbOpcodes += 8;
1093	offPage += 8;
1094	offSrc2 += 8;
1095	cbLeft -= 8;
1096	}
1097
1098	iemNativeRegFreeTmp(pReNative, idxRegSrc2Ptr);
1099	/* max cost thus far: memcmp-loop=43 vs memcmp-no-loop=30 */
1100	}
1101	/*
1102	* Otherwise, we compare with constants and merge with the general mop-up.
1103	*/
1104	else
1105	{
1106	while (cbLeft >= 8)
1107	{
1108	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr,
1109	offPage / 8);
1110	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Val,
1111	RT_MAKE_U64_FROM_MSB_U8(pbOpcodes[7], pbOpcodes[6], pbOpcodes[5], pbOpcodes[4],
1112	pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
1113	if (fPendingJmp)
1114	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1115	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, true /f64Bit/);
1116	else
1117	{
1118	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, true /f64Bit/);
1119	fPendingJmp = true;
1120	}
1121	pbOpcodes += 8;
1122	offPage += 8;
1123	cbLeft -= 8;
1124	}
1125	/* max cost thus far: 21 */
1126	}
1127
1128	/* Deal with any remaining bytes (7 or less). */
1129	Assert(cbLeft < 8);
1130	if (cbLeft >= 4)
1131	{
1132	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr,
1133	offPage / 4);
1134	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
1135	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
1136	if (fPendingJmp)
1137	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1138	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1139	else
1140	{
1141	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
1142	fPendingJmp = true;
1143	}
1144	pbOpcodes += 4;
1145	offPage += 4;
1146	cbLeft -= 4;
1147
1148	}
1149
1150	if (cbLeft >= 2)
1151	{
1152	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr,
1153	offPage / 2);
1154	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
1155	if (fPendingJmp)
1156	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1157	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1158	else
1159	{
1160	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
1161	fPendingJmp = true;
1162	}
1163	pbOpcodes += 2;
1164	offPage += 2;
1165	cbLeft -= 2;
1166	}
1167
1168	if (cbLeft > 0)
1169	{
1170	Assert(cbLeft == 1);
1171	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
1172	if (fPendingJmp)
1173	{
1174	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, pbOpcodes[0]);
1175	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1176	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1177	}
1178	else
1179	{
1180	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes[0], false /f64Bit/);
1181	fPendingJmp = true;
1182	}
1183	pbOpcodes += 1;
1184	offPage += 1;
1185	cbLeft -= 1;
1186	}
1187
1188	iemNativeRegFreeTmp(pReNative, idxRegSrc2Val);
1189	}
1190	Assert(cbLeft == 0);
1191
1192	/*
1193	* Finally, the branch on difference.
1194	*/
1195	if (fPendingJmp)
1196	off = iemNativeEmitTbExitJnz<kIemNativeLabelType_ObsoleteTb>(pReNative, off);
1197
1198	RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump);
1199
1200	/* max costs: memcmp-loop=54; memcmp-no-loop=41; only-src1-ptr=32 */
1201	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1202	iemNativeRegFreeTmp(pReNative, idxRegSrc1Val);
1203	iemNativeRegFreeTmp(pReNative, idxRegSrc1Ptr);
1204
1205	#else
1206	# error "Port me"
1207	#endif
1208	return off;
1209	}
1210
1211
1212
1213	/**
1214	* Macro that implements PC check after a conditional branch.
1215	*/
1216	#define BODY_CHECK_PC_AFTER_BRANCH(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
1217	RT_NOREF(a_cbInstr); \
1218	off = iemNativeEmitBltInCheckPcAfterBranch(pReNative, off, a_pTb, a_idxRange, a_offRange)
1219
1220	#define LIVENESS_CHECK_PC_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1221	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1222	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1223	else do { } while (0)
1224
1225	DECL_FORCE_INLINE(uint32_t)
1226	iemNativeEmitBltInCheckPcAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb,
1227	uint8_t idxRange, uint16_t offRange)
1228	{
1229	#ifdef VBOX_STRICT
1230	off = iemNativeEmitMarker(pReNative, off, 0x80000004);
1231	#endif
1232
1233	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1234	Assert(pReNative->Core.offPc == 0);
1235	#endif
1236
1237	/*
1238	* The GCPhysRangePageWithOffset value in the threaded function is a fixed
1239	* constant for us here.
1240	*
1241	* We can pretend that iem.s.cbInstrBufTotal is X86_PAGE_SIZE here, because
1242	* it serves no purpose as a CS.LIM, if that's needed we've just performed
1243	* it, and as long as we don't implement code TLB reload code here there is
1244	* no point in checking that the TLB data we're using is still valid.
1245	*
1246	* What we to do is.
1247	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1248	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1249	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1250	* we're in the wrong spot and need to find a new TB.
1251	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1252	* GCPhysRangePageWithOffset constant mentioned above.
1253	*
1254	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1255	* in 64-bit code or flat 32-bit.
1256	*/
1257
1258	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1259	the temp register, so we don't accidentally clobber something we'll be
1260	needing again immediately. This is why we get idxRegCsBase here. */
1261	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1262	kIemNativeGstRegUse_ReadOnly);
1263	uint8_t const idxRegCsBase = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX
1264	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1265	kIemNativeGstRegUse_ReadOnly);
1266
1267	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
1268
1269	#ifdef VBOX_STRICT
1270	/* Do assertions before idxRegTmp contains anything. */
1271	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1272	# ifdef RT_ARCH_AMD64
1273	{
1274	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1275	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1276	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1277	{
1278	/* cmp r/m64, imm8 */
1279	pbCodeBuf[off++] = X86_OP_REX_W;
1280	pbCodeBuf[off++] = 0x83;
1281	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1282	pbCodeBuf[off++] = 0;
1283	/* je rel8 */
1284	pbCodeBuf[off++] = 0x74;
1285	pbCodeBuf[off++] = 1;
1286	/* int3 */
1287	pbCodeBuf[off++] = 0xcc;
1288
1289	}
1290
1291	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1292	/* test r/m64, imm32 */
1293	pbCodeBuf[off++] = X86_OP_REX_W;
1294	pbCodeBuf[off++] = 0xf7;
1295	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1296	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1297	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1298	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1299	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1300	/* jz rel8 */
1301	pbCodeBuf[off++] = 0x74;
1302	pbCodeBuf[off++] = 1;
1303	/* int3 */
1304	pbCodeBuf[off++] = 0xcc;
1305	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1306	}
1307	# else
1308
1309	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1310	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1311	{
1312	off = iemNativeEmitLoadGprWithGstRegT<kIemNativeGstReg_CsBase>(pReNative, off, idxRegTmp);
1313	# ifdef RT_ARCH_ARM64
1314	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1315	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1316	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2004);
1317	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1318	# else
1319	# error "Port me!"
1320	# endif
1321	}
1322	# endif
1323
1324	#endif /* VBOX_STRICT */
1325
1326	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1327	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1328	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1329	{
1330	#ifdef RT_ARCH_ARM64
1331	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1332	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1333	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1334	#else
1335	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1336	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1337	#endif
1338	}
1339	else
1340	{
1341	#ifdef RT_ARCH_ARM64
1342	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1343	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1344	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1345	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1346	#else
1347	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1348	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1349	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1350	#endif
1351	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1352	}
1353	iemNativeRegFreeTmp(pReNative, idxRegPc);
1354
1355	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
1356	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1357	off = iemNativeEmitTbExitJa<kIemNativeLabelType_CheckBranchMiss>(pReNative, off);
1358
1359	/* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
1360	#ifdef RT_ARCH_AMD64
1361	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1362	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1363	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1364	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1365	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1366
1367	#elif defined(RT_ARCH_ARM64)
1368	uint8_t const idxRegTmp2 = iemNativeRegAllocTmp(pReNative, &off);
1369
1370	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1371	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1372	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1373	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1374
1375	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1376	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1377	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1378	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1379	# endif
1380	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1381	#else
1382	# error "Port me"
1383	#endif
1384
1385	RTGCPHYS const GCPhysRangePageWithOffset = ( iemTbGetRangePhysPageAddr(pTb, idxRange)
1386	\| pTb->aRanges[idxRange].offPhysPage)
1387	+ offRange;
1388	off = iemNativeEmitTbExitIfGprNotEqualImm<kIemNativeLabelType_CheckBranchMiss>(pReNative, off, idxRegTmp,
1389	GCPhysRangePageWithOffset);
1390
1391	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1392	return off;
1393	}
1394
1395
1396	/**
1397	* Macro that implements TLB loading and updating pbInstrBuf updating for an
1398	* instruction crossing into a new page.
1399	*
1400	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1401	*/
1402	#define BODY_LOAD_TLB_FOR_NEW_PAGE(a_pTb, a_offInstr, a_idxRange, a_cbInstr) \
1403	RT_NOREF(a_cbInstr); \
1404	off = iemNativeEmitBltLoadTlbForNewPage(pReNative, off, pTb, a_idxRange, a_offInstr)
1405
1406	#define LIVENESS_LOAD_TLB_FOR_NEW_PAGE(a_pOutgoing, a_pCallEntry) \
1407	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1408	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1409	else do { } while (0)
1410
1411	DECL_FORCE_INLINE(uint32_t)
1412	iemNativeEmitBltLoadTlbForNewPage(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint8_t offInstr)
1413	{
1414	#ifdef VBOX_STRICT
1415	off = iemNativeEmitMarker(pReNative, off, 0x80000005);
1416	#endif
1417
1418	/*
1419	* Define labels and allocate the register for holding the GCPhys of the new page.
1420	*/
1421	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1422	uint32_t const idxRegGCPhys = iemNativeRegAllocTmp(pReNative, &off);
1423	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, IEM_F_MODE_X86_IS_FLAT(pReNative->fExec), &off);
1424	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1425	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1426	: UINT32_MAX;
1427
1428	//off = iemNativeEmitBrk(pReNative, off, 0x1111);
1429
1430	/*
1431	* Jump to the TLB lookup code.
1432	*/
1433	if (!TlbState.fSkip)
1434	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1435
1436	/*
1437	* TlbMiss:
1438	*
1439	* Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
1440	*/
1441	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1442
1443	/* Save variables in volatile registers. */
1444	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegGCPhys);
1445	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1446
1447	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1448	/* Do delayed EFLAGS calculations. There are no restrictions on volatile registers here. */
1449	off = iemNativeDoPostponedEFlagsAtTlbMiss<0>(pReNative, off, &TlbState, fHstRegsNotToSave);
1450	#endif
1451
1452	/* IEMNATIVE_CALL_ARG1_GREG = offInstr */
1453	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offInstr);
1454
1455	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1456	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1457
1458	/* Done setting up parameters, make the call. */
1459	off = iemNativeEmitCallImm<true /a_fSkipEflChecks/>(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMissWithOff);
1460
1461	/* Move the result to the right register. */
1462	if (idxRegGCPhys != IEMNATIVE_CALL_RET_GREG)
1463	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegGCPhys, IEMNATIVE_CALL_RET_GREG);
1464
1465	/* Restore variables and guest shadow registers to volatile registers. */
1466	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1467	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows(true /fCode/));
1468
1469	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1470	if (!TlbState.fSkip)
1471	{
1472	/* end of TlbMiss - Jump to the done label. */
1473	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1474	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1475
1476	/*
1477	* TlbLookup:
1478	*/
1479	off = iemNativeEmitTlbLookup<false, 1 /cbMem/, 0 /fAlignMask/,
1480	IEM_ACCESS_TYPE_EXEC>(pReNative, off, &TlbState,
1481	IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX : X86_SREG_CS,
1482	idxLabelTlbLookup, idxLabelTlbMiss, idxRegGCPhys, offInstr);
1483
1484	# ifdef IEM_WITH_TLB_STATISTICS
1485	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1486	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPageWithOffset));
1487	# endif
1488
1489	/*
1490	* TlbDone:
1491	*/
1492	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1493	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1494	}
1495	#else
1496	RT_NOREF(idxLabelTlbMiss);
1497	#endif
1498
1499	/*
1500	* Now check the physical address of the page matches the expected one.
1501	*/
1502	RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
1503	off = iemNativeEmitTbExitIfGprNotEqualImm<kIemNativeLabelType_ObsoleteTb>(pReNative, off, idxRegGCPhys, GCPhysNewPage);
1504
1505	iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
1506	return off;
1507	}
1508
1509
1510	/**
1511	* Macro that implements TLB loading and updating pbInstrBuf updating when
1512	* branching or when crossing a page on an instruction boundrary.
1513	*
1514	* This differs from BODY_LOAD_TLB_FOR_NEW_PAGE in that it will first check if
1515	* it is an inter-page branch and also check the page offset.
1516	*
1517	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1518	*/
1519	#define BODY_LOAD_TLB_AFTER_BRANCH(a_pTb, a_idxRange, a_cbInstr) \
1520	RT_NOREF(a_cbInstr); \
1521	off = iemNativeEmitBltLoadTlbAfterBranch(pReNative, off, pTb, a_idxRange)
1522
1523	#define LIVENESS_LOAD_TLB_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1524	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1525	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1526	else do { } while (0)
1527
1528	DECL_FORCE_INLINE(uint32_t)
1529	iemNativeEmitBltLoadTlbAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange)
1530	{
1531	#ifdef VBOX_STRICT
1532	off = iemNativeEmitMarker(pReNative, off, 0x80000006);
1533	#endif
1534
1535	BODY_FLUSH_PENDING_WRITES();
1536
1537	/*
1538	* Define labels and allocate the register for holding the GCPhys of the new page.
1539	*/
1540	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1541	RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(pTb, idxRange)
1542	\| pTb->aRanges[idxRange].offPhysPage;
1543
1544	/*
1545	*
1546	* First check if RIP is within the current code.
1547	*
1548	* This is very similar to iemNativeEmitBltInCheckPcAfterBranch, the only
1549	* difference is what we do when stuff doesn't match up.
1550	*
1551	* What we to do is.
1552	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1553	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1554	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1555	* we need to retranslate RIP via the TLB.
1556	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1557	* GCPhysRangePageWithOffset constant mentioned above.
1558	*
1559	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1560	* in 64-bit code or flat 32-bit.
1561	*
1562	*/
1563
1564	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1565	the temp register, so we don't accidentally clobber something we'll be
1566	needing again immediately. This is why we get idxRegCsBase here.
1567	Update: We share registers with the TlbState, as the TLB code path has
1568	little in common with the rest of the code. */
1569	bool const fIsFlat = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec);
1570	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, fIsFlat, &off);
1571	uint8_t const idxRegPc = !TlbState.fSkip ? TlbState.idxRegPtr
1572	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1573	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1574	uint8_t const idxRegCsBase = !TlbState.fSkip \|\| fIsFlat ? TlbState.idxRegSegBase
1575	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1576	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1577
1578	uint8_t const idxRegTmp = !TlbState.fSkip ? TlbState.idxReg1 : iemNativeRegAllocTmp(pReNative, &off);
1579	uint8_t const idxRegTmp2 = !TlbState.fSkip ? TlbState.idxReg2 : iemNativeRegAllocTmp(pReNative, &off);
1580	uint8_t const idxRegDummy = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
1581
1582	#ifdef VBOX_STRICT
1583	/* Do assertions before idxRegTmp contains anything. */
1584	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1585	# ifdef RT_ARCH_AMD64
1586	{
1587	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1588	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1589	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1590	{
1591	/* cmp r/m64, imm8 */
1592	pbCodeBuf[off++] = X86_OP_REX_W;
1593	pbCodeBuf[off++] = 0x83;
1594	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1595	pbCodeBuf[off++] = 0;
1596	/* je rel8 */
1597	pbCodeBuf[off++] = 0x74;
1598	pbCodeBuf[off++] = 1;
1599	/* int3 */
1600	pbCodeBuf[off++] = 0xcc;
1601
1602	}
1603
1604	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1605	/* test r/m64, imm32 */
1606	pbCodeBuf[off++] = X86_OP_REX_W;
1607	pbCodeBuf[off++] = 0xf7;
1608	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1609	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1610	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1611	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1612	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1613	/* jz rel8 */
1614	pbCodeBuf[off++] = 0x74;
1615	pbCodeBuf[off++] = 1;
1616	/* int3 */
1617	pbCodeBuf[off++] = 0xcc;
1618	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1619	}
1620	# else
1621
1622	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1623	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1624	{
1625	off = iemNativeEmitLoadGprWithGstRegT<kIemNativeGstReg_CsBase>(pReNative, off, idxRegTmp);
1626	# ifdef RT_ARCH_ARM64
1627	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1628	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1629	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2006);
1630	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1631	# else
1632	# error "Port me!"
1633	# endif
1634	}
1635	# endif
1636
1637	#endif /* VBOX_STRICT */
1638
1639	/* Because we're lazy, we'll jump back here to recalc 'off' and share the
1640	GCPhysRangePageWithOffset check. This is a little risky, so we use the
1641	2nd register to check if we've looped more than once already.*/
1642	off = iemNativeEmitGprZero(pReNative, off, idxRegTmp2);
1643
1644	uint32_t const offLabelRedoChecks = off;
1645
1646	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1647	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1648	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1649	{
1650	#ifdef RT_ARCH_ARM64
1651	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1652	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1653	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1654	#else
1655	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1656	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1657	#endif
1658	}
1659	else
1660	{
1661	#ifdef RT_ARCH_ARM64
1662	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1663	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1664	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1665	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1666	#else
1667	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1668	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1669	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1670	#endif
1671	}
1672
1673	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal.
1674	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll jump to the TLB loading if this fails. */
1675	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1676	uint32_t const offFixedJumpToTlbLoad = off;
1677	off = iemNativeEmitJaToFixed(pReNative, off, off /* (ASSUME ja rel8 suffices) */);
1678
1679	/* 4a. Add iem.s.GCPhysInstrBuf to off ... */
1680	#ifdef RT_ARCH_AMD64
1681	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1682	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1683	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1684	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1685	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1686
1687	#elif defined(RT_ARCH_ARM64)
1688
1689	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1690	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1691	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1692	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1693
1694	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1695	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1696	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1697	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1698	# endif
1699	#else
1700	# error "Port me"
1701	#endif
1702
1703	/* 4b. ... and compare with GCPhysRangePageWithOffset.
1704
1705	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll have to be more
1706	careful and avoid implicit temporary register usage here.
1707
1708	Unlike the threaded version of this code, we do not obsolete TBs here to
1709	reduce the code size and because indirect calls may legally end at the
1710	same offset in two different pages depending on the program state. */
1711	/** @todo synch the threaded BODY_LOAD_TLB_AFTER_BRANCH version with this. */
1712	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
1713	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
1714	off = iemNativeEmitTbExitJnz<kIemNativeLabelType_CheckBranchMiss>(pReNative, off);
1715	uint32_t const offFixedJumpToEnd = off;
1716	off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
1717
1718	/*
1719	* TlbLoad:
1720	*
1721	* First we try to go via the TLB.
1722	*/
1723	iemNativeFixupFixedJump(pReNative, offFixedJumpToTlbLoad, off);
1724
1725	/* Check that we haven't been here before. */
1726	off = iemNativeEmitTbExitIfGprIsNotZero<kIemNativeLabelType_CheckBranchMiss>(pReNative, off, idxRegTmp2, false /f64Bit/);
1727
1728	/* Jump to the TLB lookup code. */
1729	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1730	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1731	: UINT32_MAX;
1732	//off = iemNativeEmitBrk(pReNative, off, 0x1234);
1733	if (!TlbState.fSkip)
1734	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1735
1736	/*
1737	* TlbMiss:
1738	*
1739	* Call iemNativeHlpMemCodeNewPageTlbMiss to do the work.
1740	*/
1741	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1742	RT_NOREF(idxLabelTlbMiss);
1743
1744	/* Save variables in volatile registers. */
1745	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegTmp) \| RT_BIT_32(idxRegTmp2)
1746	\| (idxRegDummy != UINT8_MAX ? RT_BIT_32(idxRegDummy) : 0);
1747	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1748
1749	#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
1750	/* Do delayed EFLAGS calculations. There are no restrictions on volatile registers here. */
1751	off = iemNativeDoPostponedEFlagsAtTlbMiss<0>(pReNative, off, &TlbState, fHstRegsNotToSave);
1752	#endif
1753
1754	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1755	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1756
1757	/* Done setting up parameters, make the call. */
1758	off = iemNativeEmitCallImm<true /a_fSkipEflChecks/>(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMiss);
1759
1760	/* Restore variables and guest shadow registers to volatile registers. */
1761	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1762	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off,
1763	TlbState.getActiveRegsWithShadows()
1764	\| RT_BIT_32(idxRegPc)
1765	\| (idxRegCsBase != UINT8_MAX ? RT_BIT_32(idxRegCsBase) : 0));
1766
1767	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1768	if (!TlbState.fSkip)
1769	{
1770	/* end of TlbMiss - Jump to the done label. */
1771	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1772	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1773
1774	/*
1775	* TlbLookup:
1776	*/
1777	off = iemNativeEmitTlbLookup<false, 1 /cbMem/, 0 /fAlignMask/,
1778	IEM_ACCESS_TYPE_EXEC, true>(pReNative, off, &TlbState, fIsFlat ? UINT8_MAX : X86_SREG_CS,
1779	idxLabelTlbLookup, idxLabelTlbMiss, idxRegDummy);
1780
1781	# ifdef IEM_WITH_TLB_STATISTICS
1782	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1783	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPage));
1784	# endif
1785
1786	/*
1787	* TlbDone:
1788	*/
1789	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1790	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1791	}
1792	#else
1793	RT_NOREF(idxLabelTlbMiss);
1794	#endif
1795
1796	/* Jmp back to the start and redo the checks. */
1797	off = iemNativeEmitLoadGpr8Imm(pReNative, off, idxRegTmp2, 1); /* indicate that we've looped once already */
1798	off = iemNativeEmitJmpToFixed(pReNative, off, offLabelRedoChecks);
1799
1800	/*
1801	* End:
1802	*
1803	* The end.
1804	*/
1805	iemNativeFixupFixedJump(pReNative, offFixedJumpToEnd, off);
1806
1807	if (!TlbState.fSkip)
1808	iemNativeRegFreeTmp(pReNative, idxRegDummy);
1809	else
1810	{
1811	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1812	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1813	iemNativeRegFreeTmp(pReNative, idxRegPc);
1814	if (idxRegCsBase != UINT8_MAX)
1815	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1816	}
1817	return off;
1818	}
1819
1820
1821	#ifdef BODY_CHECK_CS_LIM
1822	/**
1823	* Built-in function that checks the EIP/IP + uParam0 is within CS.LIM,
1824	* raising a \#GP(0) if this isn't the case.
1825	*/
1826	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLim)
1827	{
1828	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1829	BODY_SET_CUR_INSTR();
1830	BODY_FLUSH_PENDING_WRITES();
1831	BODY_CHECK_CS_LIM(cbInstr);
1832	return off;
1833	}
1834
1835	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLim)
1836	{
1837	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1838	LIVENESS_CHECK_CS_LIM(pOutgoing);
1839	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1840	RT_NOREF(pCallEntry);
1841	}
1842	#endif
1843
1844
1845	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_CS_LIM)
1846	/**
1847	* Built-in function for re-checking opcodes and CS.LIM after an instruction
1848	* that may have modified them.
1849	*/
1850	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodes)
1851	{
1852	PCIEMTB const pTb = pReNative->pTbOrg;
1853	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1854	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1855	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1856	BODY_SET_CUR_INSTR();
1857	BODY_FLUSH_PENDING_WRITES();
1858	BODY_CHECK_CS_LIM(cbInstr);
1859	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1860	return off;
1861	}
1862
1863	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodes)
1864	{
1865	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1866	LIVENESS_CHECK_CS_LIM(pOutgoing);
1867	LIVENESS_CHECK_OPCODES(pOutgoing);
1868	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1869	RT_NOREF(pCallEntry);
1870	}
1871	#endif
1872
1873
1874	#if defined(BODY_CHECK_OPCODES)
1875	/**
1876	* Built-in function for re-checking opcodes after an instruction that may have
1877	* modified them.
1878	*/
1879	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodes)
1880	{
1881	PCIEMTB const pTb = pReNative->pTbOrg;
1882	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1883	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1884	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1885	BODY_SET_CUR_INSTR();
1886	BODY_FLUSH_PENDING_WRITES();
1887	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1888	return off;
1889	}
1890
1891	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodes)
1892	{
1893	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1894	LIVENESS_CHECK_OPCODES(pOutgoing);
1895	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1896	RT_NOREF(pCallEntry);
1897	}
1898	#endif
1899
1900
1901	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1902	/**
1903	* Built-in function for re-checking opcodes and considering the need for CS.LIM
1904	* checking after an instruction that may have modified them.
1905	*/
1906	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesConsiderCsLim)
1907	{
1908	PCIEMTB const pTb = pReNative->pTbOrg;
1909	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1910	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1911	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1912	BODY_SET_CUR_INSTR();
1913	BODY_FLUSH_PENDING_WRITES();
1914	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1915	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1916	return off;
1917	}
1918
1919	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesConsiderCsLim)
1920	{
1921	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1922	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1923	LIVENESS_CHECK_OPCODES(pOutgoing);
1924	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1925	RT_NOREF(pCallEntry);
1926	}
1927	#endif
1928
1929
1930	/*
1931	* Post-branching checkers.
1932	*/
1933
1934	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1935	/**
1936	* Built-in function for checking CS.LIM, checking the PC and checking opcodes
1937	* after conditional branching within the same page.
1938	*
1939	* @see iemThreadedFunc_BltIn_CheckPcAndOpcodes
1940	*/
1941	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1942	{
1943	PCIEMTB const pTb = pReNative->pTbOrg;
1944	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1945	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1946	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1947	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1948	BODY_SET_CUR_INSTR();
1949	BODY_FLUSH_PENDING_WRITES();
1950	BODY_CHECK_CS_LIM(cbInstr);
1951	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1952	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1953	//LogFunc(("okay\n"));
1954	return off;
1955	}
1956
1957	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1958	{
1959	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1960	LIVENESS_CHECK_CS_LIM(pOutgoing);
1961	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1962	LIVENESS_CHECK_OPCODES(pOutgoing);
1963	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1964	RT_NOREF(pCallEntry);
1965	}
1966	#endif
1967
1968
1969	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH)
1970	/**
1971	* Built-in function for checking the PC and checking opcodes after conditional
1972	* branching within the same page.
1973	*
1974	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1975	*/
1976	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodes)
1977	{
1978	PCIEMTB const pTb = pReNative->pTbOrg;
1979	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1980	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1981	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1982	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1983	BODY_SET_CUR_INSTR();
1984	BODY_FLUSH_PENDING_WRITES();
1985	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1986	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1987	//LogFunc(("okay\n"));
1988	return off;
1989	}
1990
1991	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodes)
1992	{
1993	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1994	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1995	LIVENESS_CHECK_OPCODES(pOutgoing);
1996	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1997	RT_NOREF(pCallEntry);
1998	}
1999	#endif
2000
2001
2002	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2003	/**
2004	* Built-in function for checking the PC and checking opcodes and considering
2005	* the need for CS.LIM checking after conditional branching within the same
2006	* page.
2007	*
2008	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
2009	*/
2010	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
2011	{
2012	PCIEMTB const pTb = pReNative->pTbOrg;
2013	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2014	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2015	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
2016	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
2017	BODY_SET_CUR_INSTR();
2018	BODY_FLUSH_PENDING_WRITES();
2019	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2020	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
2021	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
2022	//LogFunc(("okay\n"));
2023	return off;
2024	}
2025
2026	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
2027	{
2028	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2029	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2030	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
2031	LIVENESS_CHECK_OPCODES(pOutgoing);
2032	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2033	RT_NOREF(pCallEntry);
2034	}
2035	#endif
2036
2037
2038	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
2039	/**
2040	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2041	* transitioning to a different code page.
2042	*
2043	* The code page transition can either be natural over onto the next page (with
2044	* the instruction starting at page offset zero) or by means of branching.
2045	*
2046	* @see iemThreadedFunc_BltIn_CheckOpcodesLoadingTlb
2047	*/
2048	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
2049	{
2050	PCIEMTB const pTb = pReNative->pTbOrg;
2051	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2052	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2053	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
2054	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
2055	BODY_SET_CUR_INSTR();
2056	BODY_FLUSH_PENDING_WRITES();
2057	BODY_CHECK_CS_LIM(cbInstr);
2058	Assert(offRange == 0);
2059	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
2060	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
2061	//LogFunc(("okay\n"));
2062	return off;
2063	}
2064
2065	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
2066	{
2067	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2068	LIVENESS_CHECK_CS_LIM(pOutgoing);
2069	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
2070	LIVENESS_CHECK_OPCODES(pOutgoing);
2071	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2072	RT_NOREF(pCallEntry);
2073	}
2074	#endif
2075
2076
2077	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH)
2078	/**
2079	* Built-in function for loading TLB and checking opcodes when transitioning to
2080	* a different code page.
2081	*
2082	* The code page transition can either be natural over onto the next page (with
2083	* the instruction starting at page offset zero) or by means of branching.
2084	*
2085	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
2086	*/
2087	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlb)
2088	{
2089	PCIEMTB const pTb = pReNative->pTbOrg;
2090	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2091	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2092	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
2093	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
2094	BODY_SET_CUR_INSTR();
2095	BODY_FLUSH_PENDING_WRITES();
2096	Assert(offRange == 0);
2097	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
2098	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
2099	//LogFunc(("okay\n"));
2100	return off;
2101	}
2102
2103	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlb)
2104	{
2105	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2106	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
2107	LIVENESS_CHECK_OPCODES(pOutgoing);
2108	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2109	RT_NOREF(pCallEntry);
2110	}
2111	#endif
2112
2113
2114	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2115	/**
2116	* Built-in function for loading TLB and checking opcodes and considering the
2117	* need for CS.LIM checking when transitioning to a different code page.
2118	*
2119	* The code page transition can either be natural over onto the next page (with
2120	* the instruction starting at page offset zero) or by means of branching.
2121	*
2122	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
2123	*/
2124	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
2125	{
2126	PCIEMTB const pTb = pReNative->pTbOrg;
2127	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2128	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2129	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
2130	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
2131	BODY_SET_CUR_INSTR();
2132	BODY_FLUSH_PENDING_WRITES();
2133	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2134	Assert(offRange == 0);
2135	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
2136	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
2137	//LogFunc(("okay\n"));
2138	return off;
2139	}
2140
2141	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
2142	{
2143	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2144	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2145	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
2146	LIVENESS_CHECK_OPCODES(pOutgoing);
2147	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2148	RT_NOREF(pCallEntry);
2149	}
2150	#endif
2151
2152
2153
2154	/*
2155	* Natural page crossing checkers.
2156	*/
2157
2158	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2159	/**
2160	* Built-in function for checking CS.LIM, loading TLB and checking opcodes on
2161	* both pages when transitioning to a different code page.
2162	*
2163	* This is used when the previous instruction requires revalidation of opcodes
2164	* bytes and the current instruction stries a page boundrary with opcode bytes
2165	* in both the old and new page.
2166	*
2167	* @see iemThreadedFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb
2168	*/
2169	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
2170	{
2171	PCIEMTB const pTb = pReNative->pTbOrg;
2172	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2173	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2174	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2175	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2176	uint32_t const idxRange2 = idxRange1 + 1;
2177	BODY_SET_CUR_INSTR();
2178	BODY_FLUSH_PENDING_WRITES();
2179	BODY_CHECK_CS_LIM(cbInstr);
2180	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2181	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2182	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2183	return off;
2184	}
2185
2186	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
2187	{
2188	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2189	LIVENESS_CHECK_CS_LIM(pOutgoing);
2190	LIVENESS_CHECK_OPCODES(pOutgoing);
2191	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2192	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2193	RT_NOREF(pCallEntry);
2194	}
2195	#endif
2196
2197
2198	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2199	/**
2200	* Built-in function for loading TLB and checking opcodes on both pages when
2201	* transitioning to a different code page.
2202	*
2203	* This is used when the previous instruction requires revalidation of opcodes
2204	* bytes and the current instruction stries a page boundrary with opcode bytes
2205	* in both the old and new page.
2206	*
2207	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2208	*/
2209	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2210	{
2211	PCIEMTB const pTb = pReNative->pTbOrg;
2212	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2213	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2214	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2215	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2216	uint32_t const idxRange2 = idxRange1 + 1;
2217	BODY_SET_CUR_INSTR();
2218	BODY_FLUSH_PENDING_WRITES();
2219	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2220	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2221	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2222	return off;
2223	}
2224
2225	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2226	{
2227	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2228	LIVENESS_CHECK_OPCODES(pOutgoing);
2229	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2230	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2231	RT_NOREF(pCallEntry);
2232	}
2233	#endif
2234
2235
2236	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2237	/**
2238	* Built-in function for loading TLB and checking opcodes on both pages and
2239	* considering the need for CS.LIM checking when transitioning to a different
2240	* code page.
2241	*
2242	* This is used when the previous instruction requires revalidation of opcodes
2243	* bytes and the current instruction stries a page boundrary with opcode bytes
2244	* in both the old and new page.
2245	*
2246	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2247	*/
2248	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2249	{
2250	PCIEMTB const pTb = pReNative->pTbOrg;
2251	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2252	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2253	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2254	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2255	uint32_t const idxRange2 = idxRange1 + 1;
2256	BODY_SET_CUR_INSTR();
2257	BODY_FLUSH_PENDING_WRITES();
2258	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2259	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2260	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2261	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2262	return off;
2263	}
2264
2265	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2266	{
2267	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2268	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2269	LIVENESS_CHECK_OPCODES(pOutgoing);
2270	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2271	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2272	RT_NOREF(pCallEntry);
2273	}
2274	#endif
2275
2276
2277	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2278	/**
2279	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2280	* advancing naturally to a different code page.
2281	*
2282	* Only opcodes on the new page is checked.
2283	*
2284	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb
2285	*/
2286	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2287	{
2288	PCIEMTB const pTb = pReNative->pTbOrg;
2289	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2290	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2291	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2292	//uint32_t const offRange1 = (uint32_t)uParam2;
2293	uint32_t const idxRange2 = idxRange1 + 1;
2294	BODY_SET_CUR_INSTR();
2295	BODY_FLUSH_PENDING_WRITES();
2296	BODY_CHECK_CS_LIM(cbInstr);
2297	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2298	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2299	return off;
2300	}
2301
2302	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2303	{
2304	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2305	LIVENESS_CHECK_CS_LIM(pOutgoing);
2306	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2307	LIVENESS_CHECK_OPCODES(pOutgoing);
2308	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2309	RT_NOREF(pCallEntry);
2310	}
2311	#endif
2312
2313
2314	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2315	/**
2316	* Built-in function for loading TLB and checking opcodes when advancing
2317	* naturally to a different code page.
2318	*
2319	* Only opcodes on the new page is checked.
2320	*
2321	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2322	*/
2323	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2324	{
2325	PCIEMTB const pTb = pReNative->pTbOrg;
2326	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2327	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2328	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2329	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2330	uint32_t const idxRange2 = idxRange1 + 1;
2331	BODY_SET_CUR_INSTR();
2332	BODY_FLUSH_PENDING_WRITES();
2333	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2334	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2335	return off;
2336	}
2337
2338	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2339	{
2340	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2341	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2342	LIVENESS_CHECK_OPCODES(pOutgoing);
2343	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2344	RT_NOREF(pCallEntry);
2345	}
2346	#endif
2347
2348
2349	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2350	/**
2351	* Built-in function for loading TLB and checking opcodes and considering the
2352	* need for CS.LIM checking when advancing naturally to a different code page.
2353	*
2354	* Only opcodes on the new page is checked.
2355	*
2356	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2357	*/
2358	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2359	{
2360	PCIEMTB const pTb = pReNative->pTbOrg;
2361	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2362	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2363	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2364	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2365	uint32_t const idxRange2 = idxRange1 + 1;
2366	BODY_SET_CUR_INSTR();
2367	BODY_FLUSH_PENDING_WRITES();
2368	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2369	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2370	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2371	return off;
2372	}
2373
2374	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2375	{
2376	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2377	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2378	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2379	LIVENESS_CHECK_OPCODES(pOutgoing);
2380	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2381	RT_NOREF(pCallEntry);
2382	}
2383	#endif
2384
2385
2386	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2387	/**
2388	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2389	* advancing naturally to a different code page with first instr at byte 0.
2390	*
2391	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb
2392	*/
2393	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2394	{
2395	PCIEMTB const pTb = pReNative->pTbOrg;
2396	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2397	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2398	BODY_SET_CUR_INSTR();
2399	BODY_FLUSH_PENDING_WRITES();
2400	BODY_CHECK_CS_LIM(cbInstr);
2401	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2402	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2403	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2404	return off;
2405	}
2406
2407	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2408	{
2409	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2410	LIVENESS_CHECK_CS_LIM(pOutgoing);
2411	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2412	LIVENESS_CHECK_OPCODES(pOutgoing);
2413	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2414	RT_NOREF(pCallEntry);
2415	}
2416	#endif
2417
2418
2419	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2420	/**
2421	* Built-in function for loading TLB and checking opcodes when advancing
2422	* naturally to a different code page with first instr at byte 0.
2423	*
2424	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2425	*/
2426	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2427	{
2428	PCIEMTB const pTb = pReNative->pTbOrg;
2429	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2430	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2431	BODY_SET_CUR_INSTR();
2432	BODY_FLUSH_PENDING_WRITES();
2433	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2434	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2435	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2436	return off;
2437	}
2438
2439	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2440	{
2441	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2442	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2443	LIVENESS_CHECK_OPCODES(pOutgoing);
2444	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2445	RT_NOREF(pCallEntry);
2446	}
2447	#endif
2448
2449
2450	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2451	/**
2452	* Built-in function for loading TLB and checking opcodes and considering the
2453	* need for CS.LIM checking when advancing naturally to a different code page
2454	* with first instr at byte 0.
2455	*
2456	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2457	*/
2458	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2459	{
2460	PCIEMTB const pTb = pReNative->pTbOrg;
2461	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2462	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2463	BODY_SET_CUR_INSTR();
2464	BODY_FLUSH_PENDING_WRITES();
2465	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2466	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2467	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2468	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2469	return off;
2470	}
2471
2472	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2473	{
2474	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2475	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2476	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2477	LIVENESS_CHECK_OPCODES(pOutgoing);
2478	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2479	RT_NOREF(pCallEntry);
2480	}
2481	#endif
2482
2483
2484	/**
2485	* Built-in function for jumping in the call sequence.
2486	*/
2487	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Jump)
2488	{
2489	PCIEMTB const pTb = pReNative->pTbOrg;
2490	Assert(pCallEntry->auParams[1] == 0 && pCallEntry->auParams[2] == 0);
2491	Assert(pCallEntry->auParams[0] < pTb->Thrd.cCalls);
2492	#if 1
2493	RT_NOREF(pCallEntry, pTb);
2494
2495	# ifdef VBOX_WITH_STATISTICS
2496	/* Increment StatNativeTbExitLoopFullTb. */
2497	uint32_t const offStat = RT_UOFFSETOF(VMCPU, iem.s.StatNativeTbExitLoopFullTb);
2498	# ifdef RT_ARCH_AMD64
2499	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, UINT8_MAX, UINT8_MAX, offStat);
2500	# else
2501	uint8_t const idxStatsTmp1 = iemNativeRegAllocTmp(pReNative, &off);
2502	uint8_t const idxStatsTmp2 = iemNativeRegAllocTmp(pReNative, &off);
2503	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, idxStatsTmp1, idxStatsTmp2, offStat);
2504	iemNativeRegFreeTmp(pReNative, idxStatsTmp1);
2505	iemNativeRegFreeTmp(pReNative, idxStatsTmp2);
2506	# endif
2507	# endif
2508	# ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
2509	/** @todo
2510	off = iemNativeEmitAddU32CounterInVCpuEx(pReNative, off, pTb->cInstructions, RT_UOFFSETOF(VMCPUCC, iem.s.cInstructions));
2511	*/
2512	# endif
2513
2514	/* Jump to the start of the TB. */
2515	uint32_t idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_LoopJumpTarget);
2516	AssertStmt(idxLabel < pReNative->cLabels, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_LABEL_IPE_6)); /** @todo better status */
2517	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2518	#else
2519	RT_NOREF(pReNative, pCallEntry, pTb);
2520	return off;
2521	#endif
2522	}
2523
2524	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Jump)
2525	{
2526	/* We could also use UNUSED here, but this'll is equivialent (at the moment). */
2527	IEM_LIVENESS_RAW_INIT_WITH_CALL(pOutgoing, pIncoming);
2528	RT_NOREF(pCallEntry);
2529	}
2530

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp

Download in other formats: