IEMAll.cpp@ 96622

Last change on this file since 96622 was 96494, checked in by vboxsync, 2 years ago
VMM: updates. [warning fix] bugref:10282
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 456.6 KB

Line
1	/* $Id: IEMAll.cpp 96494 2022-08-25 14:55:35Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/** @page pg_iem IEM - Interpreted Execution Manager
30	*
31	* The interpreted exeuction manager (IEM) is for executing short guest code
32	* sequences that are causing too many exits / virtualization traps. It will
33	* also be used to interpret single instructions, thus replacing the selective
34	* interpreters in EM and IOM.
35	*
36	* Design goals:
37	* - Relatively small footprint, although we favour speed and correctness
38	* over size.
39	* - Reasonably fast.
40	* - Correctly handle lock prefixed instructions.
41	* - Complete instruction set - eventually.
42	* - Refactorable into a recompiler, maybe.
43	* - Replace EMInterpret*.
44	*
45	* Using the existing disassembler has been considered, however this is thought
46	* to conflict with speed as the disassembler chews things a bit too much while
47	* leaving us with a somewhat complicated state to interpret afterwards.
48	*
49	*
50	* The current code is very much work in progress. You've been warned!
51	*
52	*
53	* @section sec_iem_fpu_instr FPU Instructions
54	*
55	* On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
56	* same or equivalent instructions on the host FPU. To make life easy, we also
57	* let the FPU prioritize the unmasked exceptions for us. This however, only
58	* works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
59	* for FPU exception delivery, because with CR0.NE=0 there is a window where we
60	* can trigger spurious FPU exceptions.
61	*
62	* The guest FPU state is not loaded into the host CPU and kept there till we
63	* leave IEM because the calling conventions have declared an all year open
64	* season on much of the FPU state. For instance an innocent looking call to
65	* memcpy might end up using a whole bunch of XMM or MM registers if the
66	* particular implementation finds it worthwhile.
67	*
68	*
69	* @section sec_iem_logging Logging
70	*
71	* The IEM code uses the \"IEM\" log group for the main logging. The different
72	* logging levels/flags are generally used for the following purposes:
73	* - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
74	* - Flow (LogFlow) : Basic enter/exit IEM state info.
75	* - Level 2 (Log2) : ?
76	* - Level 3 (Log3) : More detailed enter/exit IEM state info.
77	* - Level 4 (Log4) : Decoding mnemonics w/ EIP.
78	* - Level 5 (Log5) : Decoding details.
79	* - Level 6 (Log6) : Enables/disables the lockstep comparison with REM.
80	* - Level 7 (Log7) : iret++ execution logging.
81	* - Level 8 (Log8) : Memory writes.
82	* - Level 9 (Log9) : Memory reads.
83	* - Level 10 (Log10): TLBs.
84	* - Level 11 (Log11): Unmasked FPU exceptions.
85	*/
86
87	/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */
88	#ifdef _MSC_VER
89	# pragma warning(disable:4505)
90	#endif
91
92
93	/*********************************************************************************************************************************
94	* Header Files *
95	*********************************************************************************************************************************/
96	#define LOG_GROUP LOG_GROUP_IEM
97	#define VMCPU_INCL_CPUM_GST_CTX
98	#include <VBox/vmm/iem.h>
99	#include <VBox/vmm/cpum.h>
100	#include <VBox/vmm/apic.h>
101	#include <VBox/vmm/pdm.h>
102	#include <VBox/vmm/pgm.h>
103	#include <VBox/vmm/iom.h>
104	#include <VBox/vmm/em.h>
105	#include <VBox/vmm/hm.h>
106	#include <VBox/vmm/nem.h>
107	#include <VBox/vmm/gim.h>
108	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
109	# include <VBox/vmm/em.h>
110	# include <VBox/vmm/hm_svm.h>
111	#endif
112	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
113	# include <VBox/vmm/hmvmxinline.h>
114	#endif
115	#include <VBox/vmm/tm.h>
116	#include <VBox/vmm/dbgf.h>
117	#include <VBox/vmm/dbgftrace.h>
118	#include "IEMInternal.h"
119	#include <VBox/vmm/vmcc.h>
120	#include <VBox/log.h>
121	#include <VBox/err.h>
122	#include <VBox/param.h>
123	#include <VBox/dis.h>
124	#include <VBox/disopcode.h>
125	#include <iprt/asm-math.h>
126	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
127	# include <iprt/asm-amd64-x86.h>
128	#elif defined(RT_ARCH_ARM64) \|\| defined(RT_ARCH_ARM32)
129	# include <iprt/asm-arm.h>
130	#endif
131	#include <iprt/assert.h>
132	#include <iprt/string.h>
133	#include <iprt/x86.h>
134
135	#include "IEMInline.h"
136
137
138	/*********************************************************************************************************************************
139	* Structures and Typedefs *
140	*********************************************************************************************************************************/
141	/**
142	* CPU exception classes.
143	*/
144	typedef enum IEMXCPTCLASS
145	{
146	IEMXCPTCLASS_BENIGN,
147	IEMXCPTCLASS_CONTRIBUTORY,
148	IEMXCPTCLASS_PAGE_FAULT,
149	IEMXCPTCLASS_DOUBLE_FAULT
150	} IEMXCPTCLASS;
151
152
153	/*********************************************************************************************************************************
154	* Global Variables *
155	*********************************************************************************************************************************/
156	#if defined(IEM_LOG_MEMORY_WRITES)
157	/** What IEM just wrote. */
158	uint8_t g_abIemWrote[256];
159	/** How much IEM just wrote. */
160	size_t g_cbIemWrote;
161	#endif
162
163
164	/*********************************************************************************************************************************
165	* Internal Functions *
166	*********************************************************************************************************************************/
167	static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
168	uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT;
169
170
171	/**
172	* Initializes the decoder state.
173	*
174	* iemReInitDecoder is mostly a copy of this function.
175	*
176	* @param pVCpu The cross context virtual CPU structure of the
177	* calling thread.
178	* @param fBypassHandlers Whether to bypass access handlers.
179	* @param fDisregardLock Whether to disregard the LOCK prefix.
180	*/
181	DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock)
182	{
183	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
184	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
185	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
186	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
187	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
188	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
189	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
190	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
191	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
192	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
193
194	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
195	IEMMODE enmMode = iemCalcCpuMode(pVCpu);
196	pVCpu->iem.s.enmCpuMode = enmMode;
197	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
198	pVCpu->iem.s.enmEffAddrMode = enmMode;
199	if (enmMode != IEMMODE_64BIT)
200	{
201	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
202	pVCpu->iem.s.enmEffOpSize = enmMode;
203	}
204	else
205	{
206	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
207	pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
208	}
209	pVCpu->iem.s.fPrefixes = 0;
210	pVCpu->iem.s.uRexReg = 0;
211	pVCpu->iem.s.uRexB = 0;
212	pVCpu->iem.s.uRexIndex = 0;
213	pVCpu->iem.s.idxPrefix = 0;
214	pVCpu->iem.s.uVex3rdReg = 0;
215	pVCpu->iem.s.uVexLength = 0;
216	pVCpu->iem.s.fEvexStuff = 0;
217	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
218	#ifdef IEM_WITH_CODE_TLB
219	pVCpu->iem.s.pbInstrBuf = NULL;
220	pVCpu->iem.s.offInstrNextByte = 0;
221	pVCpu->iem.s.offCurInstrStart = 0;
222	# ifdef VBOX_STRICT
223	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
224	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
225	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
226	# endif
227	#else
228	pVCpu->iem.s.offOpcode = 0;
229	pVCpu->iem.s.cbOpcode = 0;
230	#endif
231	pVCpu->iem.s.offModRm = 0;
232	pVCpu->iem.s.cActiveMappings = 0;
233	pVCpu->iem.s.iNextMapping = 0;
234	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
235	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
236	pVCpu->iem.s.fDisregardLock = fDisregardLock;
237
238	#ifdef DBGFTRACE_ENABLED
239	switch (enmMode)
240	{
241	case IEMMODE_64BIT:
242	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
243	break;
244	case IEMMODE_32BIT:
245	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
246	break;
247	case IEMMODE_16BIT:
248	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
249	break;
250	}
251	#endif
252	}
253
254
255	/**
256	* Reinitializes the decoder state 2nd+ loop of IEMExecLots.
257	*
258	* This is mostly a copy of iemInitDecoder.
259	*
260	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
261	*/
262	DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu)
263	{
264	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
265	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
266	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
267	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
268	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
269	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
270	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
271	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
272	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
273
274	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu); /** @todo this should be updated during execution! */
275	IEMMODE enmMode = iemCalcCpuMode(pVCpu);
276	pVCpu->iem.s.enmCpuMode = enmMode; /** @todo this should be updated during execution! */
277	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
278	pVCpu->iem.s.enmEffAddrMode = enmMode;
279	if (enmMode != IEMMODE_64BIT)
280	{
281	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
282	pVCpu->iem.s.enmEffOpSize = enmMode;
283	}
284	else
285	{
286	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
287	pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
288	}
289	pVCpu->iem.s.fPrefixes = 0;
290	pVCpu->iem.s.uRexReg = 0;
291	pVCpu->iem.s.uRexB = 0;
292	pVCpu->iem.s.uRexIndex = 0;
293	pVCpu->iem.s.idxPrefix = 0;
294	pVCpu->iem.s.uVex3rdReg = 0;
295	pVCpu->iem.s.uVexLength = 0;
296	pVCpu->iem.s.fEvexStuff = 0;
297	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
298	#ifdef IEM_WITH_CODE_TLB
299	if (pVCpu->iem.s.pbInstrBuf)
300	{
301	uint64_t off = (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rip : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)
302	- pVCpu->iem.s.uInstrBufPc;
303	if (off < pVCpu->iem.s.cbInstrBufTotal)
304	{
305	pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
306	pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
307	if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
308	pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
309	else
310	pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
311	}
312	else
313	{
314	pVCpu->iem.s.pbInstrBuf = NULL;
315	pVCpu->iem.s.offInstrNextByte = 0;
316	pVCpu->iem.s.offCurInstrStart = 0;
317	pVCpu->iem.s.cbInstrBuf = 0;
318	pVCpu->iem.s.cbInstrBufTotal = 0;
319	}
320	}
321	else
322	{
323	pVCpu->iem.s.offInstrNextByte = 0;
324	pVCpu->iem.s.offCurInstrStart = 0;
325	pVCpu->iem.s.cbInstrBuf = 0;
326	pVCpu->iem.s.cbInstrBufTotal = 0;
327	}
328	#else
329	pVCpu->iem.s.cbOpcode = 0;
330	pVCpu->iem.s.offOpcode = 0;
331	#endif
332	pVCpu->iem.s.offModRm = 0;
333	Assert(pVCpu->iem.s.cActiveMappings == 0);
334	pVCpu->iem.s.iNextMapping = 0;
335	Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
336	Assert(pVCpu->iem.s.fBypassHandlers == false);
337
338	#ifdef DBGFTRACE_ENABLED
339	switch (enmMode)
340	{
341	case IEMMODE_64BIT:
342	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
343	break;
344	case IEMMODE_32BIT:
345	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
346	break;
347	case IEMMODE_16BIT:
348	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
349	break;
350	}
351	#endif
352	}
353
354
355
356	/**
357	* Prefetch opcodes the first time when starting executing.
358	*
359	* @returns Strict VBox status code.
360	* @param pVCpu The cross context virtual CPU structure of the
361	* calling thread.
362	* @param fBypassHandlers Whether to bypass access handlers.
363	* @param fDisregardLock Whether to disregard LOCK prefixes.
364	*
365	* @todo Combine fDisregardLock and fBypassHandlers into a flag parameter and
366	* store them as such.
367	*/
368	static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock) RT_NOEXCEPT
369	{
370	iemInitDecoder(pVCpu, fBypassHandlers, fDisregardLock);
371
372	#ifdef IEM_WITH_CODE_TLB
373	/** @todo Do ITLB lookup here. */
374
375	#else /* !IEM_WITH_CODE_TLB */
376
377	/*
378	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
379	*
380	* First translate CS:rIP to a physical address.
381	*/
382	uint32_t cbToTryRead;
383	RTGCPTR GCPtrPC;
384	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
385	{
386	cbToTryRead = GUEST_PAGE_SIZE;
387	GCPtrPC = pVCpu->cpum.GstCtx.rip;
388	if (IEM_IS_CANONICAL(GCPtrPC))
389	cbToTryRead = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
390	else
391	return iemRaiseGeneralProtectionFault0(pVCpu);
392	}
393	else
394	{
395	uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
396	AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) \|\| pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
397	if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
398	cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
399	else
400	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
401	if (cbToTryRead) { /* likely */ }
402	else /* overflowed */
403	{
404	Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
405	cbToTryRead = UINT32_MAX;
406	}
407	GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
408	Assert(GCPtrPC <= UINT32_MAX);
409	}
410
411	PGMPTWALK Walk;
412	int rc = PGMGstGetPage(pVCpu, GCPtrPC, &Walk);
413	if (RT_SUCCESS(rc))
414	Assert(Walk.fSucceeded); /* probable. */
415	else
416	{
417	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
418	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
419	if (Walk.fFailed & PGM_WALKFAIL_EPT)
420	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
421	#endif
422	return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
423	}
424	if ((Walk.fEffective & X86_PTE_US) \|\| pVCpu->iem.s.uCpl != 3) { /* likely */ }
425	else
426	{
427	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
428	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
429	if (Walk.fFailed & PGM_WALKFAIL_EPT)
430	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
431	#endif
432	return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
433	}
434	if (!(Walk.fEffective & X86_PTE_PAE_NX) \|\| !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
435	else
436	{
437	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
438	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
439	if (Walk.fFailed & PGM_WALKFAIL_EPT)
440	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
441	#endif
442	return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
443	}
444	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
445	/** @todo Check reserved bits and such stuff. PGM is better at doing
446	* that, so do it when implementing the guest virtual address
447	* TLB... */
448
449	/*
450	* Read the bytes at this address.
451	*/
452	uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
453	if (cbToTryRead > cbLeftOnPage)
454	cbToTryRead = cbLeftOnPage;
455	if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
456	cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
457
458	if (!pVCpu->iem.s.fBypassHandlers)
459	{
460	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
461	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
462	{ /* likely */ }
463	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
464	{
465	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
466	GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
467	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
468	}
469	else
470	{
471	Log((RT_SUCCESS(rcStrict)
472	? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
473	: "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
474	GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
475	return rcStrict;
476	}
477	}
478	else
479	{
480	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
481	if (RT_SUCCESS(rc))
482	{ /* likely */ }
483	else
484	{
485	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
486	GCPtrPC, GCPhys, rc, cbToTryRead));
487	return rc;
488	}
489	}
490	pVCpu->iem.s.cbOpcode = cbToTryRead;
491	#endif /* !IEM_WITH_CODE_TLB */
492	return VINF_SUCCESS;
493	}
494
495
496	/**
497	* Invalidates the IEM TLBs.
498	*
499	* This is called internally as well as by PGM when moving GC mappings.
500	*
501	* @returns
502	* @param pVCpu The cross context virtual CPU structure of the calling
503	* thread.
504	*/
505	VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu)
506	{
507	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
508	Log10(("IEMTlbInvalidateAll\n"));
509	# ifdef IEM_WITH_CODE_TLB
510	pVCpu->iem.s.cbInstrBufTotal = 0;
511	pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR;
512	if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0)
513	{ /* very likely */ }
514	else
515	{
516	pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR;
517	unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
518	while (i-- > 0)
519	pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0;
520	}
521	# endif
522
523	# ifdef IEM_WITH_DATA_TLB
524	pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR;
525	if (pVCpu->iem.s.DataTlb.uTlbRevision != 0)
526	{ /* very likely */ }
527	else
528	{
529	pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR;
530	unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
531	while (i-- > 0)
532	pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0;
533	}
534	# endif
535	#else
536	RT_NOREF(pVCpu);
537	#endif
538	}
539
540
541	/**
542	* Invalidates a page in the TLBs.
543	*
544	* @param pVCpu The cross context virtual CPU structure of the calling
545	* thread.
546	* @param GCPtr The address of the page to invalidate
547	* @thread EMT(pVCpu)
548	*/
549	VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr)
550	{
551	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
552	Log10(("IEMTlbInvalidatePage: GCPtr=%RGv\n", GCPtr));
553	GCPtr = IEMTLB_CALC_TAG_NO_REV(GCPtr);
554	Assert(!(GCPtr >> (48 - X86_PAGE_SHIFT)));
555	uintptr_t const idx = IEMTLB_TAG_TO_INDEX(GCPtr);
556
557	# ifdef IEM_WITH_CODE_TLB
558	if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr \| pVCpu->iem.s.CodeTlb.uTlbRevision))
559	{
560	pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0;
561	if (GCPtr == IEMTLB_CALC_TAG_NO_REV(pVCpu->iem.s.uInstrBufPc))
562	pVCpu->iem.s.cbInstrBufTotal = 0;
563	}
564	# endif
565
566	# ifdef IEM_WITH_DATA_TLB
567	if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr \| pVCpu->iem.s.DataTlb.uTlbRevision))
568	pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0;
569	# endif
570	#else
571	NOREF(pVCpu); NOREF(GCPtr);
572	#endif
573	}
574
575
576	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
577	/**
578	* Invalid both TLBs slow fashion following a rollover.
579	*
580	* Worker for IEMTlbInvalidateAllPhysical,
581	* IEMTlbInvalidateAllPhysicalAllCpus, iemOpcodeFetchBytesJmp, iemMemMap,
582	* iemMemMapJmp and others.
583	*
584	* @thread EMT(pVCpu)
585	*/
586	static void IEMTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu)
587	{
588	Log10(("IEMTlbInvalidateAllPhysicalSlow\n"));
589	ASMAtomicWriteU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
590	ASMAtomicWriteU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
591
592	unsigned i;
593	# ifdef IEM_WITH_CODE_TLB
594	i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
595	while (i-- > 0)
596	{
597	pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3 = NULL;
598	pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ
599	\| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PHYS_REV);
600	}
601	# endif
602	# ifdef IEM_WITH_DATA_TLB
603	i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
604	while (i-- > 0)
605	{
606	pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3 = NULL;
607	pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ
608	\| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PHYS_REV);
609	}
610	# endif
611
612	}
613	#endif
614
615
616	/**
617	* Invalidates the host physical aspects of the IEM TLBs.
618	*
619	* This is called internally as well as by PGM when moving GC mappings.
620	*
621	* @param pVCpu The cross context virtual CPU structure of the calling
622	* thread.
623	* @note Currently not used.
624	*/
625	VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu)
626	{
627	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
628	/* Note! This probably won't end up looking exactly like this, but it give an idea... */
629	Log10(("IEMTlbInvalidateAllPhysical\n"));
630
631	# ifdef IEM_WITH_CODE_TLB
632	pVCpu->iem.s.cbInstrBufTotal = 0;
633	# endif
634	uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;
635	if (RT_LIKELY(uTlbPhysRev > IEMTLB_PHYS_REV_INCR * 2))
636	{
637	pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;
638	pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;
639	}
640	else
641	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
642	#else
643	NOREF(pVCpu);
644	#endif
645	}
646
647
648	/**
649	* Invalidates the host physical aspects of the IEM TLBs.
650	*
651	* This is called internally as well as by PGM when moving GC mappings.
652	*
653	* @param pVM The cross context VM structure.
654	* @param idCpuCaller The ID of the calling EMT if available to the caller,
655	* otherwise NIL_VMCPUID.
656	*
657	* @remarks Caller holds the PGM lock.
658	*/
659	VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVMCC pVM, VMCPUID idCpuCaller)
660	{
661	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
662	PVMCPUCC const pVCpuCaller = idCpuCaller >= pVM->cCpus ? VMMGetCpu(pVM) : VMMGetCpuById(pVM, idCpuCaller);
663	if (pVCpuCaller)
664	VMCPU_ASSERT_EMT(pVCpuCaller);
665	Log10(("IEMTlbInvalidateAllPhysicalAllCpus\n"));
666
667	VMCC_FOR_EACH_VMCPU(pVM)
668	{
669	# ifdef IEM_WITH_CODE_TLB
670	if (pVCpuCaller == pVCpu)
671	pVCpu->iem.s.cbInstrBufTotal = 0;
672	# endif
673
674	uint64_t const uTlbPhysRevPrev = ASMAtomicUoReadU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev);
675	uint64_t uTlbPhysRevNew = uTlbPhysRevPrev + IEMTLB_PHYS_REV_INCR;
676	if (RT_LIKELY(uTlbPhysRevNew > IEMTLB_PHYS_REV_INCR * 2))
677	{ /* likely */}
678	else if (pVCpuCaller == pVCpu)
679	uTlbPhysRevNew = IEMTLB_PHYS_REV_INCR;
680	else
681	{
682	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
683	continue;
684	}
685	ASMAtomicCmpXchgU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
686	ASMAtomicCmpXchgU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
687	}
688	VMCC_FOR_EACH_VMCPU_END(pVM);
689
690	#else
691	RT_NOREF(pVM, idCpuCaller);
692	#endif
693	}
694
695	#ifdef IEM_WITH_CODE_TLB
696
697	/**
698	* Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
699	* failure and jumps.
700	*
701	* We end up here for a number of reasons:
702	* - pbInstrBuf isn't yet initialized.
703	* - Advancing beyond the buffer boundrary (e.g. cross page).
704	* - Advancing beyond the CS segment limit.
705	* - Fetching from non-mappable page (e.g. MMIO).
706	*
707	* @param pVCpu The cross context virtual CPU structure of the
708	* calling thread.
709	* @param pvDst Where to return the bytes.
710	* @param cbDst Number of bytes to read.
711	*
712	* @todo Make cbDst = 0 a way of initializing pbInstrBuf?
713	*/
714	void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) RT_NOEXCEPT
715	{
716	#ifdef IN_RING3
717	for (;;)
718	{
719	Assert(cbDst <= 8);
720	uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
721
722	/*
723	* We might have a partial buffer match, deal with that first to make the
724	* rest simpler. This is the first part of the cross page/buffer case.
725	*/
726	if (pVCpu->iem.s.pbInstrBuf != NULL)
727	{
728	if (offBuf < pVCpu->iem.s.cbInstrBuf)
729	{
730	Assert(offBuf + cbDst > pVCpu->iem.s.cbInstrBuf);
731	uint32_t const cbCopy = pVCpu->iem.s.cbInstrBuf - pVCpu->iem.s.offInstrNextByte;
732	memcpy(pvDst, &pVCpu->iem.s.pbInstrBuf[offBuf], cbCopy);
733
734	cbDst -= cbCopy;
735	pvDst = (uint8_t *)pvDst + cbCopy;
736	offBuf += cbCopy;
737	pVCpu->iem.s.offInstrNextByte += offBuf;
738	}
739	}
740
741	/*
742	* Check segment limit, figuring how much we're allowed to access at this point.
743	*
744	* We will fault immediately if RIP is past the segment limit / in non-canonical
745	* territory. If we do continue, there are one or more bytes to read before we
746	* end up in trouble and we need to do that first before faulting.
747	*/
748	RTGCPTR GCPtrFirst;
749	uint32_t cbMaxRead;
750	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
751	{
752	GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
753	if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
754	{ /* likely */ }
755	else
756	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
757	cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
758	}
759	else
760	{
761	GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
762	Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) \|\| pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
763	if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
764	{ /* likely */ }
765	else
766	iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
767	cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
768	if (cbMaxRead != 0)
769	{ /* likely */ }
770	else
771	{
772	/* Overflowed because address is 0 and limit is max. */
773	Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
774	cbMaxRead = X86_PAGE_SIZE;
775	}
776	GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
777	uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
778	if (cbMaxRead2 < cbMaxRead)
779	cbMaxRead = cbMaxRead2;
780	/** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
781	}
782
783	/*
784	* Get the TLB entry for this piece of code.
785	*/
786	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.CodeTlb, GCPtrFirst);
787	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.CodeTlb, uTag);
788	if (pTlbe->uTag == uTag)
789	{
790	/* likely when executing lots of code, otherwise unlikely */
791	# ifdef VBOX_WITH_STATISTICS
792	pVCpu->iem.s.CodeTlb.cTlbHits++;
793	# endif
794	}
795	else
796	{
797	pVCpu->iem.s.CodeTlb.cTlbMisses++;
798	PGMPTWALK Walk;
799	int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &Walk);
800	if (RT_FAILURE(rc))
801	{
802	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
803	/** @todo Nested VMX: Need to handle EPT violation/misconfig here? */
804	Assert(!(Walk.fFailed & PGM_WALKFAIL_EPT));
805	#endif
806	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
807	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, rc);
808	}
809
810	AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
811	Assert(Walk.fSucceeded);
812	pTlbe->uTag = uTag;
813	pTlbe->fFlagsAndPhysRev = (~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A))
814	\| (Walk.fEffective >> X86_PTE_PAE_BIT_NX);
815	pTlbe->GCPhys = Walk.GCPhys;
816	pTlbe->pbMappingR3 = NULL;
817	}
818
819	/*
820	* Check TLB page table level access flags.
821	*/
822	if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER \| IEMTLBE_F_PT_NO_EXEC))
823	{
824	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && pVCpu->iem.s.uCpl == 3)
825	{
826	Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
827	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
828	}
829	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
830	{
831	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
832	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
833	}
834	}
835
836	/*
837	* Look up the physical page info if necessary.
838	*/
839	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
840	{ /* not necessary */ }
841	else
842	{
843	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
844	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
845	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
846	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
847	if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
848	{ /* likely */ }
849	else
850	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
851	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
852	\| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_UNASSIGNED);
853	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
854	&pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
855	AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
856	}
857
858	# if defined(IN_RING3) \|\| defined(IN_RING0) /** @todo fixme */
859	/*
860	* Try do a direct read using the pbMappingR3 pointer.
861	*/
862	if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ))
863	== pVCpu->iem.s.CodeTlb.uTlbPhysRev)
864	{
865	uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
866	pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead;
867	if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
868	{
869	pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(15, cbMaxRead);
870	pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
871	}
872	else
873	{
874	uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
875	Assert(cbInstr < cbMaxRead);
876	pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
877	pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
878	}
879	if (cbDst <= cbMaxRead)
880	{
881	pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
882	pVCpu->iem.s.uInstrBufPc = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
883	pVCpu->iem.s.pbInstrBuf = pTlbe->pbMappingR3;
884	memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
885	return;
886	}
887	pVCpu->iem.s.pbInstrBuf = NULL;
888
889	memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
890	pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
891	}
892	else
893	# endif
894	#if 0
895	/*
896	* If there is no special read handling, so we can read a bit more and
897	* put it in the prefetch buffer.
898	*/
899	if ( cbDst < cbMaxRead
900	&& (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
901	{
902	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
903	&pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
904	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
905	{ /* likely */ }
906	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
907	{
908	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
909	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
910	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
911	AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICRC_VAL(rcStrict)));
912	}
913	else
914	{
915	Log((RT_SUCCESS(rcStrict)
916	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
917	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
918	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
919	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
920	}
921	}
922	/*
923	* Special read handling, so only read exactly what's needed.
924	* This is a highly unlikely scenario.
925	*/
926	else
927	#endif
928	{
929	pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++;
930	uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
931	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
932	pvDst, cbToRead, PGMACCESSORIGIN_IEM);
933	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
934	{ /* likely */ }
935	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
936	{
937	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
938	GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
939	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
940	AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict)));
941	}
942	else
943	{
944	Log((RT_SUCCESS(rcStrict)
945	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
946	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
947	GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
948	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
949	}
950	pVCpu->iem.s.offInstrNextByte = offBuf + cbToRead;
951	if (cbToRead == cbDst)
952	return;
953	}
954
955	/*
956	* More to read, loop.
957	*/
958	cbDst -= cbMaxRead;
959	pvDst = (uint8_t *)pvDst + cbMaxRead;
960	}
961	#else
962	RT_NOREF(pvDst, cbDst);
963	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VERR_INTERNAL_ERROR);
964	#endif
965	}
966
967	#else
968
969	/**
970	* Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
971	* exception if it fails.
972	*
973	* @returns Strict VBox status code.
974	* @param pVCpu The cross context virtual CPU structure of the
975	* calling thread.
976	* @param cbMin The minimum number of bytes relative offOpcode
977	* that must be read.
978	*/
979	VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT
980	{
981	/*
982	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
983	*
984	* First translate CS:rIP to a physical address.
985	*/
986	uint8_t cbLeft = pVCpu->iem.s.cbOpcode - pVCpu->iem.s.offOpcode; Assert(cbLeft < cbMin);
987	uint32_t cbToTryRead;
988	RTGCPTR GCPtrNext;
989	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
990	{
991	cbToTryRead = GUEST_PAGE_SIZE;
992	GCPtrNext = pVCpu->cpum.GstCtx.rip + pVCpu->iem.s.cbOpcode;
993	if (!IEM_IS_CANONICAL(GCPtrNext))
994	return iemRaiseGeneralProtectionFault0(pVCpu);
995	}
996	else
997	{
998	uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
999	Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) \|\| pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
1000	GCPtrNext32 += pVCpu->iem.s.cbOpcode;
1001	if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
1002	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1003	cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
1004	if (!cbToTryRead) /* overflowed */
1005	{
1006	Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1007	cbToTryRead = UINT32_MAX;
1008	/** @todo check out wrapping around the code segment. */
1009	}
1010	if (cbToTryRead < cbMin - cbLeft)
1011	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1012	GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
1013	}
1014
1015	/* Only read up to the end of the page, and make sure we don't read more
1016	than the opcode buffer can hold. */
1017	uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1018	if (cbToTryRead > cbLeftOnPage)
1019	cbToTryRead = cbLeftOnPage;
1020	if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode)
1021	cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode;
1022	/** @todo r=bird: Convert assertion into undefined opcode exception? */
1023	Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
1024
1025	PGMPTWALK Walk;
1026	int rc = PGMGstGetPage(pVCpu, GCPtrNext, &Walk);
1027	if (RT_FAILURE(rc))
1028	{
1029	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
1030	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1031	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1032	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
1033	#endif
1034	return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
1035	}
1036	if (!(Walk.fEffective & X86_PTE_US) && pVCpu->iem.s.uCpl == 3)
1037	{
1038	Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
1039	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1040	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1041	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1042	#endif
1043	return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1044	}
1045	if ((Walk.fEffective & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
1046	{
1047	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
1048	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1049	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1050	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1051	#endif
1052	return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1053	}
1054	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1055	Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pVCpu->iem.s.cbOpcode));
1056	/** @todo Check reserved bits and such stuff. PGM is better at doing
1057	* that, so do it when implementing the guest virtual address
1058	* TLB... */
1059
1060	/*
1061	* Read the bytes at this address.
1062	*
1063	* We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
1064	* and since PATM should only patch the start of an instruction there
1065	* should be no need to check again here.
1066	*/
1067	if (!pVCpu->iem.s.fBypassHandlers)
1068	{
1069	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode],
1070	cbToTryRead, PGMACCESSORIGIN_IEM);
1071	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1072	{ /* likely */ }
1073	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1074	{
1075	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1076	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1077	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1078	}
1079	else
1080	{
1081	Log((RT_SUCCESS(rcStrict)
1082	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1083	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1084	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1085	return rcStrict;
1086	}
1087	}
1088	else
1089	{
1090	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode], GCPhys, cbToTryRead);
1091	if (RT_SUCCESS(rc))
1092	{ /* likely */ }
1093	else
1094	{
1095	Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
1096	return rc;
1097	}
1098	}
1099	pVCpu->iem.s.cbOpcode += cbToTryRead;
1100	Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
1101
1102	return VINF_SUCCESS;
1103	}
1104
1105	#endif /* !IEM_WITH_CODE_TLB */
1106	#ifndef IEM_WITH_SETJMP
1107
1108	/**
1109	* Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
1110	*
1111	* @returns Strict VBox status code.
1112	* @param pVCpu The cross context virtual CPU structure of the
1113	* calling thread.
1114	* @param pb Where to return the opcode byte.
1115	*/
1116	VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT
1117	{
1118	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
1119	if (rcStrict == VINF_SUCCESS)
1120	{
1121	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1122	*pb = pVCpu->iem.s.abOpcode[offOpcode];
1123	pVCpu->iem.s.offOpcode = offOpcode + 1;
1124	}
1125	else
1126	*pb = 0;
1127	return rcStrict;
1128	}
1129
1130	#else /* IEM_WITH_SETJMP */
1131
1132	/**
1133	* Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
1134	*
1135	* @returns The opcode byte.
1136	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1137	*/
1138	uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1139	{
1140	# ifdef IEM_WITH_CODE_TLB
1141	uint8_t u8;
1142	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
1143	return u8;
1144	# else
1145	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
1146	if (rcStrict == VINF_SUCCESS)
1147	return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
1148	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1149	# endif
1150	}
1151
1152	#endif /* IEM_WITH_SETJMP */
1153
1154	#ifndef IEM_WITH_SETJMP
1155
1156	/**
1157	* Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
1158	*
1159	* @returns Strict VBox status code.
1160	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1161	* @param pu16 Where to return the opcode dword.
1162	*/
1163	VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
1164	{
1165	uint8_t u8;
1166	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1167	if (rcStrict == VINF_SUCCESS)
1168	*pu16 = (int8_t)u8;
1169	return rcStrict;
1170	}
1171
1172
1173	/**
1174	* Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
1175	*
1176	* @returns Strict VBox status code.
1177	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1178	* @param pu32 Where to return the opcode dword.
1179	*/
1180	VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1181	{
1182	uint8_t u8;
1183	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1184	if (rcStrict == VINF_SUCCESS)
1185	*pu32 = (int8_t)u8;
1186	return rcStrict;
1187	}
1188
1189
1190	/**
1191	* Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
1192	*
1193	* @returns Strict VBox status code.
1194	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1195	* @param pu64 Where to return the opcode qword.
1196	*/
1197	VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1198	{
1199	uint8_t u8;
1200	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1201	if (rcStrict == VINF_SUCCESS)
1202	*pu64 = (int8_t)u8;
1203	return rcStrict;
1204	}
1205
1206	#endif /* !IEM_WITH_SETJMP */
1207
1208
1209	#ifndef IEM_WITH_SETJMP
1210
1211	/**
1212	* Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
1213	*
1214	* @returns Strict VBox status code.
1215	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1216	* @param pu16 Where to return the opcode word.
1217	*/
1218	VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
1219	{
1220	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1221	if (rcStrict == VINF_SUCCESS)
1222	{
1223	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1224	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1225	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1226	# else
1227	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1228	# endif
1229	pVCpu->iem.s.offOpcode = offOpcode + 2;
1230	}
1231	else
1232	*pu16 = 0;
1233	return rcStrict;
1234	}
1235
1236	#else /* IEM_WITH_SETJMP */
1237
1238	/**
1239	* Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
1240	*
1241	* @returns The opcode word.
1242	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1243	*/
1244	uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1245	{
1246	# ifdef IEM_WITH_CODE_TLB
1247	uint16_t u16;
1248	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
1249	return u16;
1250	# else
1251	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1252	if (rcStrict == VINF_SUCCESS)
1253	{
1254	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1255	pVCpu->iem.s.offOpcode += 2;
1256	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1257	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1258	# else
1259	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1260	# endif
1261	}
1262	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1263	# endif
1264	}
1265
1266	#endif /* IEM_WITH_SETJMP */
1267
1268	#ifndef IEM_WITH_SETJMP
1269
1270	/**
1271	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
1272	*
1273	* @returns Strict VBox status code.
1274	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1275	* @param pu32 Where to return the opcode double word.
1276	*/
1277	VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1278	{
1279	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1280	if (rcStrict == VINF_SUCCESS)
1281	{
1282	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1283	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1284	pVCpu->iem.s.offOpcode = offOpcode + 2;
1285	}
1286	else
1287	*pu32 = 0;
1288	return rcStrict;
1289	}
1290
1291
1292	/**
1293	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
1294	*
1295	* @returns Strict VBox status code.
1296	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1297	* @param pu64 Where to return the opcode quad word.
1298	*/
1299	VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1300	{
1301	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1302	if (rcStrict == VINF_SUCCESS)
1303	{
1304	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1305	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1306	pVCpu->iem.s.offOpcode = offOpcode + 2;
1307	}
1308	else
1309	*pu64 = 0;
1310	return rcStrict;
1311	}
1312
1313	#endif /* !IEM_WITH_SETJMP */
1314
1315	#ifndef IEM_WITH_SETJMP
1316
1317	/**
1318	* Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
1319	*
1320	* @returns Strict VBox status code.
1321	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1322	* @param pu32 Where to return the opcode dword.
1323	*/
1324	VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1325	{
1326	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1327	if (rcStrict == VINF_SUCCESS)
1328	{
1329	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1330	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1331	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1332	# else
1333	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1334	pVCpu->iem.s.abOpcode[offOpcode + 1],
1335	pVCpu->iem.s.abOpcode[offOpcode + 2],
1336	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1337	# endif
1338	pVCpu->iem.s.offOpcode = offOpcode + 4;
1339	}
1340	else
1341	*pu32 = 0;
1342	return rcStrict;
1343	}
1344
1345	#else /* IEM_WITH_SETJMP */
1346
1347	/**
1348	* Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
1349	*
1350	* @returns The opcode dword.
1351	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1352	*/
1353	uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1354	{
1355	# ifdef IEM_WITH_CODE_TLB
1356	uint32_t u32;
1357	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
1358	return u32;
1359	# else
1360	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1361	if (rcStrict == VINF_SUCCESS)
1362	{
1363	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1364	pVCpu->iem.s.offOpcode = offOpcode + 4;
1365	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1366	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1367	# else
1368	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1369	pVCpu->iem.s.abOpcode[offOpcode + 1],
1370	pVCpu->iem.s.abOpcode[offOpcode + 2],
1371	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1372	# endif
1373	}
1374	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1375	# endif
1376	}
1377
1378	#endif /* IEM_WITH_SETJMP */
1379
1380	#ifndef IEM_WITH_SETJMP
1381
1382	/**
1383	* Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
1384	*
1385	* @returns Strict VBox status code.
1386	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1387	* @param pu64 Where to return the opcode dword.
1388	*/
1389	VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1390	{
1391	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1392	if (rcStrict == VINF_SUCCESS)
1393	{
1394	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1395	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1396	pVCpu->iem.s.abOpcode[offOpcode + 1],
1397	pVCpu->iem.s.abOpcode[offOpcode + 2],
1398	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1399	pVCpu->iem.s.offOpcode = offOpcode + 4;
1400	}
1401	else
1402	*pu64 = 0;
1403	return rcStrict;
1404	}
1405
1406
1407	/**
1408	* Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
1409	*
1410	* @returns Strict VBox status code.
1411	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1412	* @param pu64 Where to return the opcode qword.
1413	*/
1414	VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1415	{
1416	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1417	if (rcStrict == VINF_SUCCESS)
1418	{
1419	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1420	*pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1421	pVCpu->iem.s.abOpcode[offOpcode + 1],
1422	pVCpu->iem.s.abOpcode[offOpcode + 2],
1423	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1424	pVCpu->iem.s.offOpcode = offOpcode + 4;
1425	}
1426	else
1427	*pu64 = 0;
1428	return rcStrict;
1429	}
1430
1431	#endif /* !IEM_WITH_SETJMP */
1432
1433	#ifndef IEM_WITH_SETJMP
1434
1435	/**
1436	* Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
1437	*
1438	* @returns Strict VBox status code.
1439	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1440	* @param pu64 Where to return the opcode qword.
1441	*/
1442	VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1443	{
1444	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
1445	if (rcStrict == VINF_SUCCESS)
1446	{
1447	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1448	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1449	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1450	# else
1451	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1452	pVCpu->iem.s.abOpcode[offOpcode + 1],
1453	pVCpu->iem.s.abOpcode[offOpcode + 2],
1454	pVCpu->iem.s.abOpcode[offOpcode + 3],
1455	pVCpu->iem.s.abOpcode[offOpcode + 4],
1456	pVCpu->iem.s.abOpcode[offOpcode + 5],
1457	pVCpu->iem.s.abOpcode[offOpcode + 6],
1458	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1459	# endif
1460	pVCpu->iem.s.offOpcode = offOpcode + 8;
1461	}
1462	else
1463	*pu64 = 0;
1464	return rcStrict;
1465	}
1466
1467	#else /* IEM_WITH_SETJMP */
1468
1469	/**
1470	* Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
1471	*
1472	* @returns The opcode qword.
1473	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1474	*/
1475	uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1476	{
1477	# ifdef IEM_WITH_CODE_TLB
1478	uint64_t u64;
1479	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
1480	return u64;
1481	# else
1482	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
1483	if (rcStrict == VINF_SUCCESS)
1484	{
1485	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1486	pVCpu->iem.s.offOpcode = offOpcode + 8;
1487	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1488	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1489	# else
1490	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1491	pVCpu->iem.s.abOpcode[offOpcode + 1],
1492	pVCpu->iem.s.abOpcode[offOpcode + 2],
1493	pVCpu->iem.s.abOpcode[offOpcode + 3],
1494	pVCpu->iem.s.abOpcode[offOpcode + 4],
1495	pVCpu->iem.s.abOpcode[offOpcode + 5],
1496	pVCpu->iem.s.abOpcode[offOpcode + 6],
1497	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1498	# endif
1499	}
1500	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1501	# endif
1502	}
1503
1504	#endif /* IEM_WITH_SETJMP */
1505
1506
1507
1508	/** @name Misc Worker Functions.
1509	* @{
1510	*/
1511
1512	/**
1513	* Gets the exception class for the specified exception vector.
1514	*
1515	* @returns The class of the specified exception.
1516	* @param uVector The exception vector.
1517	*/
1518	static IEMXCPTCLASS iemGetXcptClass(uint8_t uVector) RT_NOEXCEPT
1519	{
1520	Assert(uVector <= X86_XCPT_LAST);
1521	switch (uVector)
1522	{
1523	case X86_XCPT_DE:
1524	case X86_XCPT_TS:
1525	case X86_XCPT_NP:
1526	case X86_XCPT_SS:
1527	case X86_XCPT_GP:
1528	case X86_XCPT_SX: /* AMD only */
1529	return IEMXCPTCLASS_CONTRIBUTORY;
1530
1531	case X86_XCPT_PF:
1532	case X86_XCPT_VE: /* Intel only */
1533	return IEMXCPTCLASS_PAGE_FAULT;
1534
1535	case X86_XCPT_DF:
1536	return IEMXCPTCLASS_DOUBLE_FAULT;
1537	}
1538	return IEMXCPTCLASS_BENIGN;
1539	}
1540
1541
1542	/**
1543	* Evaluates how to handle an exception caused during delivery of another event
1544	* (exception / interrupt).
1545	*
1546	* @returns How to handle the recursive exception.
1547	* @param pVCpu The cross context virtual CPU structure of the
1548	* calling thread.
1549	* @param fPrevFlags The flags of the previous event.
1550	* @param uPrevVector The vector of the previous event.
1551	* @param fCurFlags The flags of the current exception.
1552	* @param uCurVector The vector of the current exception.
1553	* @param pfXcptRaiseInfo Where to store additional information about the
1554	* exception condition. Optional.
1555	*/
1556	VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPUCC pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags,
1557	uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo)
1558	{
1559	/*
1560	* Only CPU exceptions can be raised while delivering other events, software interrupt
1561	* (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception.
1562	*/
1563	AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID);
1564	Assert(pVCpu); RT_NOREF(pVCpu);
1565	Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector));
1566
1567	IEMXCPTRAISE enmRaise = IEMXCPTRAISE_CURRENT_XCPT;
1568	IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE;
1569	if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1570	{
1571	IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector);
1572	if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN)
1573	{
1574	IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector);
1575	if ( enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT
1576	&& ( enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT
1577	\|\| enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY))
1578	{
1579	enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
1580	fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF
1581	: IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT;
1582	Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector,
1583	uCurVector, pVCpu->cpum.GstCtx.cr2));
1584	}
1585	else if ( enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY
1586	&& enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY)
1587	{
1588	enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
1589	Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector));
1590	}
1591	else if ( enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT
1592	&& ( enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY
1593	\|\| enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT))
1594	{
1595	enmRaise = IEMXCPTRAISE_TRIPLE_FAULT;
1596	Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector));
1597	}
1598	}
1599	else
1600	{
1601	if (uPrevVector == X86_XCPT_NMI)
1602	{
1603	fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT;
1604	if (uCurVector == X86_XCPT_PF)
1605	{
1606	fRaiseInfo \|= IEMXCPTRAISEINFO_NMI_PF;
1607	Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n"));
1608	}
1609	}
1610	else if ( uPrevVector == X86_XCPT_AC
1611	&& uCurVector == X86_XCPT_AC)
1612	{
1613	enmRaise = IEMXCPTRAISE_CPU_HANG;
1614	fRaiseInfo = IEMXCPTRAISEINFO_AC_AC;
1615	Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n"));
1616	}
1617	}
1618	}
1619	else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT)
1620	{
1621	fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT;
1622	if (uCurVector == X86_XCPT_PF)
1623	fRaiseInfo \|= IEMXCPTRAISEINFO_EXT_INT_PF;
1624	}
1625	else
1626	{
1627	Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT);
1628	fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT;
1629	}
1630
1631	if (pfXcptRaiseInfo)
1632	*pfXcptRaiseInfo = fRaiseInfo;
1633	return enmRaise;
1634	}
1635
1636
1637	/**
1638	* Enters the CPU shutdown state initiated by a triple fault or other
1639	* unrecoverable conditions.
1640	*
1641	* @returns Strict VBox status code.
1642	* @param pVCpu The cross context virtual CPU structure of the
1643	* calling thread.
1644	*/
1645	static VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPUCC pVCpu) RT_NOEXCEPT
1646	{
1647	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
1648	IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
1649
1650	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
1651	{
1652	Log2(("shutdown: Guest intercept -> #VMEXIT\n"));
1653	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
1654	}
1655
1656	RT_NOREF(pVCpu);
1657	return VINF_EM_TRIPLE_FAULT;
1658	}
1659
1660
1661	/**
1662	* Validates a new SS segment.
1663	*
1664	* @returns VBox strict status code.
1665	* @param pVCpu The cross context virtual CPU structure of the
1666	* calling thread.
1667	* @param NewSS The new SS selctor.
1668	* @param uCpl The CPL to load the stack for.
1669	* @param pDesc Where to return the descriptor.
1670	*/
1671	static VBOXSTRICTRC iemMiscValidateNewSS(PVMCPUCC pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc) RT_NOEXCEPT
1672	{
1673	/* Null selectors are not allowed (we're not called for dispatching
1674	interrupts with SS=0 in long mode). */
1675	if (!(NewSS & X86_SEL_MASK_OFF_RPL))
1676	{
1677	Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS));
1678	return iemRaiseTaskSwitchFault0(pVCpu);
1679	}
1680
1681	/** @todo testcase: check that the TSS.ssX RPL is checked. Also check when. */
1682	if ((NewSS & X86_SEL_RPL) != uCpl)
1683	{
1684	Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
1685	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1686	}
1687
1688	/*
1689	* Read the descriptor.
1690	*/
1691	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS);
1692	if (rcStrict != VINF_SUCCESS)
1693	return rcStrict;
1694
1695	/*
1696	* Perform the descriptor validation documented for LSS, POP SS and MOV SS.
1697	*/
1698	if (!pDesc->Legacy.Gen.u1DescType)
1699	{
1700	Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1701	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1702	}
1703
1704	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1705	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1706	{
1707	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1708	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1709	}
1710	if (pDesc->Legacy.Gen.u2Dpl != uCpl)
1711	{
1712	Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
1713	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1714	}
1715
1716	/* Is it there? */
1717	/** @todo testcase: Is this checked before the canonical / limit check below? */
1718	if (!pDesc->Legacy.Gen.u1Present)
1719	{
1720	Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
1721	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS);
1722	}
1723
1724	return VINF_SUCCESS;
1725	}
1726
1727	/** @} */
1728
1729
1730	/** @name Raising Exceptions.
1731	*
1732	* @{
1733	*/
1734
1735
1736	/**
1737	* Loads the specified stack far pointer from the TSS.
1738	*
1739	* @returns VBox strict status code.
1740	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1741	* @param uCpl The CPL to load the stack for.
1742	* @param pSelSS Where to return the new stack segment.
1743	* @param puEsp Where to return the new stack pointer.
1744	*/
1745	static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPUCC pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp) RT_NOEXCEPT
1746	{
1747	VBOXSTRICTRC rcStrict;
1748	Assert(uCpl < 4);
1749
1750	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
1751	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
1752	{
1753	/*
1754	* 16-bit TSS (X86TSS16).
1755	*/
1756	case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
1757	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1758	{
1759	uint32_t off = uCpl * 4 + 2;
1760	if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit)
1761	{
1762	/** @todo check actual access pattern here. */
1763	uint32_t u32Tmp = 0; /* gcc maybe... */
1764	rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1765	if (rcStrict == VINF_SUCCESS)
1766	{
1767	*puEsp = RT_LOWORD(u32Tmp);
1768	*pSelSS = RT_HIWORD(u32Tmp);
1769	return VINF_SUCCESS;
1770	}
1771	}
1772	else
1773	{
1774	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
1775	rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1776	}
1777	break;
1778	}
1779
1780	/*
1781	* 32-bit TSS (X86TSS32).
1782	*/
1783	case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
1784	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1785	{
1786	uint32_t off = uCpl * 8 + 4;
1787	if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit)
1788	{
1789	/** @todo check actual access pattern here. */
1790	uint64_t u64Tmp;
1791	rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1792	if (rcStrict == VINF_SUCCESS)
1793	{
1794	*puEsp = u64Tmp & UINT32_MAX;
1795	*pSelSS = (RTSEL)(u64Tmp >> 32);
1796	return VINF_SUCCESS;
1797	}
1798	}
1799	else
1800	{
1801	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
1802	rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1803	}
1804	break;
1805	}
1806
1807	default:
1808	AssertFailed();
1809	rcStrict = VERR_IEM_IPE_4;
1810	break;
1811	}
1812
1813	puEsp = 0; / make gcc happy */
1814	pSelSS = 0; / make gcc happy */
1815	return rcStrict;
1816	}
1817
1818
1819	/**
1820	* Loads the specified stack pointer from the 64-bit TSS.
1821	*
1822	* @returns VBox strict status code.
1823	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1824	* @param uCpl The CPL to load the stack for.
1825	* @param uIst The interrupt stack table index, 0 if to use uCpl.
1826	* @param puRsp Where to return the new stack pointer.
1827	*/
1828	static VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPUCC pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp) RT_NOEXCEPT
1829	{
1830	Assert(uCpl < 4);
1831	Assert(uIst < 8);
1832	puRsp = 0; / make gcc happy */
1833
1834	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
1835	AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5);
1836
1837	uint32_t off;
1838	if (uIst)
1839	off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1);
1840	else
1841	off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0);
1842	if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit)
1843	{
1844	Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit));
1845	return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1846	}
1847
1848	return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1849	}
1850
1851
1852	/**
1853	* Adjust the CPU state according to the exception being raised.
1854	*
1855	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1856	* @param u8Vector The exception that has been raised.
1857	*/
1858	DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPUCC pVCpu, uint8_t u8Vector)
1859	{
1860	switch (u8Vector)
1861	{
1862	case X86_XCPT_DB:
1863	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
1864	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
1865	break;
1866	/** @todo Read the AMD and Intel exception reference... */
1867	}
1868	}
1869
1870
1871	/**
1872	* Implements exceptions and interrupts for real mode.
1873	*
1874	* @returns VBox strict status code.
1875	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1876	* @param cbInstr The number of bytes to offset rIP by in the return
1877	* address.
1878	* @param u8Vector The interrupt / exception vector number.
1879	* @param fFlags The flags.
1880	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
1881	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
1882	*/
1883	static VBOXSTRICTRC
1884	iemRaiseXcptOrIntInRealMode(PVMCPUCC pVCpu,
1885	uint8_t cbInstr,
1886	uint8_t u8Vector,
1887	uint32_t fFlags,
1888	uint16_t uErr,
1889	uint64_t uCr2) RT_NOEXCEPT
1890	{
1891	NOREF(uErr); NOREF(uCr2);
1892	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
1893
1894	/*
1895	* Read the IDT entry.
1896	*/
1897	if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
1898	{
1899	Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
1900	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1901	}
1902	RTFAR16 Idte;
1903	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t )&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) u8Vector);
1904	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1905	{
1906	Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
1907	return rcStrict;
1908	}
1909
1910	/*
1911	* Push the stack frame.
1912	*/
1913	uint16_t *pu16Frame;
1914	uint64_t uNewRsp;
1915	rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, 3, (void **)&pu16Frame, &uNewRsp);
1916	if (rcStrict != VINF_SUCCESS)
1917	return rcStrict;
1918
1919	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
1920	#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
1921	AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
1922	if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186)
1923	fEfl \|= UINT16_C(0xf000);
1924	#endif
1925	pu16Frame[2] = (uint16_t)fEfl;
1926	pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel;
1927	pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
1928	rcStrict = iemMemStackPushCommitSpecial(pVCpu, pu16Frame, uNewRsp);
1929	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1930	return rcStrict;
1931
1932	/*
1933	* Load the vector address into cs:ip and make exception specific state
1934	* adjustments.
1935	*/
1936	pVCpu->cpum.GstCtx.cs.Sel = Idte.sel;
1937	pVCpu->cpum.GstCtx.cs.ValidSel = Idte.sel;
1938	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1939	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)Idte.sel << 4;
1940	/** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
1941	pVCpu->cpum.GstCtx.rip = Idte.off;
1942	fEfl &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_AC);
1943	IEMMISC_SET_EFL(pVCpu, fEfl);
1944
1945	/** @todo do we actually do this in real mode? */
1946	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1947	iemRaiseXcptAdjustState(pVCpu, u8Vector);
1948
1949	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
1950	}
1951
1952
1953	/**
1954	* Loads a NULL data selector into when coming from V8086 mode.
1955	*
1956	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1957	* @param pSReg Pointer to the segment register.
1958	*/
1959	DECLINLINE(void) iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
1960	{
1961	pSReg->Sel = 0;
1962	pSReg->ValidSel = 0;
1963	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1964	{
1965	/* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
1966	pSReg->Attr.u &= X86DESCATTR_DT \| X86DESCATTR_TYPE \| X86DESCATTR_DPL \| X86DESCATTR_G \| X86DESCATTR_D;
1967	pSReg->Attr.u \|= X86DESCATTR_UNUSABLE;
1968	}
1969	else
1970	{
1971	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1972	/** @todo check this on AMD-V */
1973	pSReg->u64Base = 0;
1974	pSReg->u32Limit = 0;
1975	}
1976	}
1977
1978
1979	/**
1980	* Loads a segment selector during a task switch in V8086 mode.
1981	*
1982	* @param pSReg Pointer to the segment register.
1983	* @param uSel The selector value to load.
1984	*/
1985	DECLINLINE(void) iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel)
1986	{
1987	/* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
1988	pSReg->Sel = uSel;
1989	pSReg->ValidSel = uSel;
1990	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1991	pSReg->u64Base = uSel << 4;
1992	pSReg->u32Limit = 0xffff;
1993	pSReg->Attr.u = 0xf3;
1994	}
1995
1996
1997	/**
1998	* Loads a segment selector during a task switch in protected mode.
1999	*
2000	* In this task switch scenario, we would throw \#TS exceptions rather than
2001	* \#GPs.
2002	*
2003	* @returns VBox strict status code.
2004	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2005	* @param pSReg Pointer to the segment register.
2006	* @param uSel The new selector value.
2007	*
2008	* @remarks This does _not_ handle CS or SS.
2009	* @remarks This expects pVCpu->iem.s.uCpl to be up to date.
2010	*/
2011	static VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPUCC pVCpu, PCPUMSELREG pSReg, uint16_t uSel) RT_NOEXCEPT
2012	{
2013	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
2014
2015	/* Null data selector. */
2016	if (!(uSel & X86_SEL_MASK_OFF_RPL))
2017	{
2018	iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel);
2019	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
2020	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2021	return VINF_SUCCESS;
2022	}
2023
2024	/* Fetch the descriptor. */
2025	IEMSELDESC Desc;
2026	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS);
2027	if (rcStrict != VINF_SUCCESS)
2028	{
2029	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
2030	VBOXSTRICTRC_VAL(rcStrict)));
2031	return rcStrict;
2032	}
2033
2034	/* Must be a data segment or readable code segment. */
2035	if ( !Desc.Legacy.Gen.u1DescType
2036	\|\| (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2037	{
2038	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
2039	Desc.Legacy.Gen.u4Type));
2040	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2041	}
2042
2043	/* Check privileges for data segments and non-conforming code segments. */
2044	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
2045	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
2046	{
2047	/* The RPL and the new CPL must be less than or equal to the DPL. */
2048	if ( (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
2049	\|\| (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl))
2050	{
2051	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
2052	uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
2053	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2054	}
2055	}
2056
2057	/* Is it there? */
2058	if (!Desc.Legacy.Gen.u1Present)
2059	{
2060	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
2061	return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2062	}
2063
2064	/* The base and limit. */
2065	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
2066	uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
2067
2068	/*
2069	* Ok, everything checked out fine. Now set the accessed bit before
2070	* committing the result into the registers.
2071	*/
2072	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2073	{
2074	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
2075	if (rcStrict != VINF_SUCCESS)
2076	return rcStrict;
2077	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2078	}
2079
2080	/* Commit */
2081	pSReg->Sel = uSel;
2082	pSReg->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
2083	pSReg->u32Limit = cbLimit;
2084	pSReg->u64Base = u64Base; /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */
2085	pSReg->ValidSel = uSel;
2086	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2087	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2088	pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
2089
2090	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
2091	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2092	return VINF_SUCCESS;
2093	}
2094
2095
2096	/**
2097	* Performs a task switch.
2098	*
2099	* If the task switch is the result of a JMP, CALL or IRET instruction, the
2100	* caller is responsible for performing the necessary checks (like DPL, TSS
2101	* present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
2102	* reference for JMP, CALL, IRET.
2103	*
2104	* If the task switch is the due to a software interrupt or hardware exception,
2105	* the caller is responsible for validating the TSS selector and descriptor. See
2106	* Intel Instruction reference for INT n.
2107	*
2108	* @returns VBox strict status code.
2109	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2110	* @param enmTaskSwitch The cause of the task switch.
2111	* @param uNextEip The EIP effective after the task switch.
2112	* @param fFlags The flags, see IEM_XCPT_FLAGS_XXX.
2113	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2114	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2115	* @param SelTSS The TSS selector of the new task.
2116	* @param pNewDescTSS Pointer to the new TSS descriptor.
2117	*/
2118	VBOXSTRICTRC
2119	iemTaskSwitch(PVMCPUCC pVCpu,
2120	IEMTASKSWITCH enmTaskSwitch,
2121	uint32_t uNextEip,
2122	uint32_t fFlags,
2123	uint16_t uErr,
2124	uint64_t uCr2,
2125	RTSEL SelTSS,
2126	PIEMSELDESC pNewDescTSS) RT_NOEXCEPT
2127	{
2128	Assert(!IEM_IS_REAL_MODE(pVCpu));
2129	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
2130	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
2131
2132	uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type;
2133	Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2134	\|\| uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2135	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2136	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2137
2138	bool const fIsNewTSS386 = ( uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2139	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2140
2141	Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTSS,
2142	fIsNewTSS386, pVCpu->cpum.GstCtx.eip, uNextEip));
2143
2144	/* Update CR2 in case it's a page-fault. */
2145	/** @todo This should probably be done much earlier in IEM/PGM. See
2146	* @bugref{5653#c49}. */
2147	if (fFlags & IEM_XCPT_FLAGS_CR2)
2148	pVCpu->cpum.GstCtx.cr2 = uCr2;
2149
2150	/*
2151	* Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
2152	* subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
2153	*/
2154	uint32_t const uNewTSSLimit = pNewDescTSS->Legacy.Gen.u16LimitLow \| (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16);
2155	uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
2156	if (uNewTSSLimit < uNewTSSLimitMin)
2157	{
2158	Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n",
2159	enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin));
2160	return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
2161	}
2162
2163	/*
2164	* Task switches in VMX non-root mode always cause task switches.
2165	* The new TSS must have been read and validated (DPL, limits etc.) before a
2166	* task-switch VM-exit commences.
2167	*
2168	* See Intel spec. 25.4.2 "Treatment of Task Switches".
2169	*/
2170	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
2171	{
2172	Log(("iemTaskSwitch: Guest intercept (source=%u, sel=%#x) -> VM-exit.\n", enmTaskSwitch, SelTSS));
2173	IEM_VMX_VMEXIT_TASK_SWITCH_RET(pVCpu, enmTaskSwitch, SelTSS, uNextEip - pVCpu->cpum.GstCtx.eip);
2174	}
2175
2176	/*
2177	* The SVM nested-guest intercept for task-switch takes priority over all exceptions
2178	* after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept".
2179	*/
2180	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
2181	{
2182	uint32_t const uExitInfo1 = SelTSS;
2183	uint32_t uExitInfo2 = uErr;
2184	switch (enmTaskSwitch)
2185	{
2186	case IEMTASKSWITCH_JUMP: uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_JUMP; break;
2187	case IEMTASKSWITCH_IRET: uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_IRET; break;
2188	default: break;
2189	}
2190	if (fFlags & IEM_XCPT_FLAGS_ERR)
2191	uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE;
2192	if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
2193	uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF;
2194
2195	Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2));
2196	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2);
2197	RT_NOREF2(uExitInfo1, uExitInfo2);
2198	}
2199
2200	/*
2201	* Check the current TSS limit. The last written byte to the current TSS during the
2202	* task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
2203	* See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2204	*
2205	* The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
2206	* end up with smaller than "legal" TSS limits.
2207	*/
2208	uint32_t const uCurTSSLimit = pVCpu->cpum.GstCtx.tr.u32Limit;
2209	uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29;
2210	if (uCurTSSLimit < uCurTSSLimitMin)
2211	{
2212	Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n",
2213	enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin));
2214	return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
2215	}
2216
2217	/*
2218	* Verify that the new TSS can be accessed and map it. Map only the required contents
2219	* and not the entire TSS.
2220	*/
2221	void *pvNewTSS;
2222	uint32_t const cbNewTSS = uNewTSSLimitMin + 1;
2223	RTGCPTR const GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy);
2224	AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
2225	/** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
2226	* not perform correct translation if this happens. See Intel spec. 7.2.1
2227	* "Task-State Segment". */
2228	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW, 0);
2229	if (rcStrict != VINF_SUCCESS)
2230	{
2231	Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch,
2232	cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict)));
2233	return rcStrict;
2234	}
2235
2236	/*
2237	* Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
2238	*/
2239	uint32_t u32EFlags = pVCpu->cpum.GstCtx.eflags.u32;
2240	if ( enmTaskSwitch == IEMTASKSWITCH_JUMP
2241	\|\| enmTaskSwitch == IEMTASKSWITCH_IRET)
2242	{
2243	PX86DESC pDescCurTSS;
2244	rcStrict = iemMemMap(pVCpu, (void *)&pDescCurTSS, sizeof(pDescCurTSS), UINT8_MAX,
2245	pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
2246	if (rcStrict != VINF_SUCCESS)
2247	{
2248	Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2249	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2250	return rcStrict;
2251	}
2252
2253	pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2254	rcStrict = iemMemCommitAndUnmap(pVCpu, pDescCurTSS, IEM_ACCESS_SYS_RW);
2255	if (rcStrict != VINF_SUCCESS)
2256	{
2257	Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2258	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2259	return rcStrict;
2260	}
2261
2262	/* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
2263	if (enmTaskSwitch == IEMTASKSWITCH_IRET)
2264	{
2265	Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2266	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2267	u32EFlags &= ~X86_EFL_NT;
2268	}
2269	}
2270
2271	/*
2272	* Save the CPU state into the current TSS.
2273	*/
2274	RTGCPTR const GCPtrCurTSS = pVCpu->cpum.GstCtx.tr.u64Base;
2275	if (GCPtrNewTSS == GCPtrCurTSS)
2276	{
2277	Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur\|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS));
2278	Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
2279	pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.eax,
2280	pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel,
2281	pVCpu->cpum.GstCtx.ldtr.Sel));
2282	}
2283	if (fIsNewTSS386)
2284	{
2285	/*
2286	* Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
2287	* See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2288	*/
2289	void *pvCurTSS32;
2290	uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS32, eip);
2291	uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip);
2292	AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
2293	rcStrict = iemMemMap(pVCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0);
2294	if (rcStrict != VINF_SUCCESS)
2295	{
2296	Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
2297	enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
2298	return rcStrict;
2299	}
2300
2301	/* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
2302	PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS);
2303	pCurTSS32->eip = uNextEip;
2304	pCurTSS32->eflags = u32EFlags;
2305	pCurTSS32->eax = pVCpu->cpum.GstCtx.eax;
2306	pCurTSS32->ecx = pVCpu->cpum.GstCtx.ecx;
2307	pCurTSS32->edx = pVCpu->cpum.GstCtx.edx;
2308	pCurTSS32->ebx = pVCpu->cpum.GstCtx.ebx;
2309	pCurTSS32->esp = pVCpu->cpum.GstCtx.esp;
2310	pCurTSS32->ebp = pVCpu->cpum.GstCtx.ebp;
2311	pCurTSS32->esi = pVCpu->cpum.GstCtx.esi;
2312	pCurTSS32->edi = pVCpu->cpum.GstCtx.edi;
2313	pCurTSS32->es = pVCpu->cpum.GstCtx.es.Sel;
2314	pCurTSS32->cs = pVCpu->cpum.GstCtx.cs.Sel;
2315	pCurTSS32->ss = pVCpu->cpum.GstCtx.ss.Sel;
2316	pCurTSS32->ds = pVCpu->cpum.GstCtx.ds.Sel;
2317	pCurTSS32->fs = pVCpu->cpum.GstCtx.fs.Sel;
2318	pCurTSS32->gs = pVCpu->cpum.GstCtx.gs.Sel;
2319
2320	rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS32, IEM_ACCESS_SYS_RW);
2321	if (rcStrict != VINF_SUCCESS)
2322	{
2323	Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2324	VBOXSTRICTRC_VAL(rcStrict)));
2325	return rcStrict;
2326	}
2327	}
2328	else
2329	{
2330	/*
2331	* Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
2332	*/
2333	void *pvCurTSS16;
2334	uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS16, ip);
2335	uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip);
2336	AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
2337	rcStrict = iemMemMap(pVCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0);
2338	if (rcStrict != VINF_SUCCESS)
2339	{
2340	Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
2341	enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
2342	return rcStrict;
2343	}
2344
2345	/* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
2346	PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS);
2347	pCurTSS16->ip = uNextEip;
2348	pCurTSS16->flags = u32EFlags;
2349	pCurTSS16->ax = pVCpu->cpum.GstCtx.ax;
2350	pCurTSS16->cx = pVCpu->cpum.GstCtx.cx;
2351	pCurTSS16->dx = pVCpu->cpum.GstCtx.dx;
2352	pCurTSS16->bx = pVCpu->cpum.GstCtx.bx;
2353	pCurTSS16->sp = pVCpu->cpum.GstCtx.sp;
2354	pCurTSS16->bp = pVCpu->cpum.GstCtx.bp;
2355	pCurTSS16->si = pVCpu->cpum.GstCtx.si;
2356	pCurTSS16->di = pVCpu->cpum.GstCtx.di;
2357	pCurTSS16->es = pVCpu->cpum.GstCtx.es.Sel;
2358	pCurTSS16->cs = pVCpu->cpum.GstCtx.cs.Sel;
2359	pCurTSS16->ss = pVCpu->cpum.GstCtx.ss.Sel;
2360	pCurTSS16->ds = pVCpu->cpum.GstCtx.ds.Sel;
2361
2362	rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS16, IEM_ACCESS_SYS_RW);
2363	if (rcStrict != VINF_SUCCESS)
2364	{
2365	Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2366	VBOXSTRICTRC_VAL(rcStrict)));
2367	return rcStrict;
2368	}
2369	}
2370
2371	/*
2372	* Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
2373	*/
2374	if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2375	\|\| enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2376	{
2377	/* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
2378	PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS;
2379	pNewTSS->selPrev = pVCpu->cpum.GstCtx.tr.Sel;
2380	}
2381
2382	/*
2383	* Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
2384	* it's done further below with error handling (e.g. CR3 changes will go through PGM).
2385	*/
2386	uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
2387	uint16_t uNewES, uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
2388	bool fNewDebugTrap;
2389	if (fIsNewTSS386)
2390	{
2391	PCX86TSS32 pNewTSS32 = (PCX86TSS32)pvNewTSS;
2392	uNewCr3 = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0;
2393	uNewEip = pNewTSS32->eip;
2394	uNewEflags = pNewTSS32->eflags;
2395	uNewEax = pNewTSS32->eax;
2396	uNewEcx = pNewTSS32->ecx;
2397	uNewEdx = pNewTSS32->edx;
2398	uNewEbx = pNewTSS32->ebx;
2399	uNewEsp = pNewTSS32->esp;
2400	uNewEbp = pNewTSS32->ebp;
2401	uNewEsi = pNewTSS32->esi;
2402	uNewEdi = pNewTSS32->edi;
2403	uNewES = pNewTSS32->es;
2404	uNewCS = pNewTSS32->cs;
2405	uNewSS = pNewTSS32->ss;
2406	uNewDS = pNewTSS32->ds;
2407	uNewFS = pNewTSS32->fs;
2408	uNewGS = pNewTSS32->gs;
2409	uNewLdt = pNewTSS32->selLdt;
2410	fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap);
2411	}
2412	else
2413	{
2414	PCX86TSS16 pNewTSS16 = (PCX86TSS16)pvNewTSS;
2415	uNewCr3 = 0;
2416	uNewEip = pNewTSS16->ip;
2417	uNewEflags = pNewTSS16->flags;
2418	uNewEax = UINT32_C(0xffff0000) \| pNewTSS16->ax;
2419	uNewEcx = UINT32_C(0xffff0000) \| pNewTSS16->cx;
2420	uNewEdx = UINT32_C(0xffff0000) \| pNewTSS16->dx;
2421	uNewEbx = UINT32_C(0xffff0000) \| pNewTSS16->bx;
2422	uNewEsp = UINT32_C(0xffff0000) \| pNewTSS16->sp;
2423	uNewEbp = UINT32_C(0xffff0000) \| pNewTSS16->bp;
2424	uNewEsi = UINT32_C(0xffff0000) \| pNewTSS16->si;
2425	uNewEdi = UINT32_C(0xffff0000) \| pNewTSS16->di;
2426	uNewES = pNewTSS16->es;
2427	uNewCS = pNewTSS16->cs;
2428	uNewSS = pNewTSS16->ss;
2429	uNewDS = pNewTSS16->ds;
2430	uNewFS = 0;
2431	uNewGS = 0;
2432	uNewLdt = pNewTSS16->selLdt;
2433	fNewDebugTrap = false;
2434	}
2435
2436	if (GCPtrNewTSS == GCPtrCurTSS)
2437	Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
2438	uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
2439
2440	/*
2441	* We're done accessing the new TSS.
2442	*/
2443	rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewTSS, IEM_ACCESS_SYS_RW);
2444	if (rcStrict != VINF_SUCCESS)
2445	{
2446	Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
2447	return rcStrict;
2448	}
2449
2450	/*
2451	* Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
2452	*/
2453	if (enmTaskSwitch != IEMTASKSWITCH_IRET)
2454	{
2455	rcStrict = iemMemMap(pVCpu, (void *)&pNewDescTSS, sizeof(pNewDescTSS), UINT8_MAX,
2456	pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
2457	if (rcStrict != VINF_SUCCESS)
2458	{
2459	Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2460	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2461	return rcStrict;
2462	}
2463
2464	/* Check that the descriptor indicates the new TSS is available (not busy). */
2465	AssertMsg( pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2466	\|\| pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
2467	("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type));
2468
2469	pNewDescTSS->Legacy.Gate.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2470	rcStrict = iemMemCommitAndUnmap(pVCpu, pNewDescTSS, IEM_ACCESS_SYS_RW);
2471	if (rcStrict != VINF_SUCCESS)
2472	{
2473	Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2474	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2475	return rcStrict;
2476	}
2477	}
2478
2479	/*
2480	* From this point on, we're technically in the new task. We will defer exceptions
2481	* until the completion of the task switch but before executing any instructions in the new task.
2482	*/
2483	pVCpu->cpum.GstCtx.tr.Sel = SelTSS;
2484	pVCpu->cpum.GstCtx.tr.ValidSel = SelTSS;
2485	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
2486	pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy);
2487	pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy);
2488	pVCpu->cpum.GstCtx.tr.u64Base = X86DESC_BASE(&pNewDescTSS->Legacy);
2489	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR);
2490
2491	/* Set the busy bit in TR. */
2492	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2493
2494	/* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
2495	if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2496	\|\| enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2497	{
2498	uNewEflags \|= X86_EFL_NT;
2499	}
2500
2501	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL; /** @todo Should we clear DR7.LE bit too? */
2502	pVCpu->cpum.GstCtx.cr0 \|= X86_CR0_TS;
2503	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0);
2504
2505	pVCpu->cpum.GstCtx.eip = uNewEip;
2506	pVCpu->cpum.GstCtx.eax = uNewEax;
2507	pVCpu->cpum.GstCtx.ecx = uNewEcx;
2508	pVCpu->cpum.GstCtx.edx = uNewEdx;
2509	pVCpu->cpum.GstCtx.ebx = uNewEbx;
2510	pVCpu->cpum.GstCtx.esp = uNewEsp;
2511	pVCpu->cpum.GstCtx.ebp = uNewEbp;
2512	pVCpu->cpum.GstCtx.esi = uNewEsi;
2513	pVCpu->cpum.GstCtx.edi = uNewEdi;
2514
2515	uNewEflags &= X86_EFL_LIVE_MASK;
2516	uNewEflags \|= X86_EFL_RA1_MASK;
2517	IEMMISC_SET_EFL(pVCpu, uNewEflags);
2518
2519	/*
2520	* Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
2521	* will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
2522	* due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
2523	*/
2524	pVCpu->cpum.GstCtx.es.Sel = uNewES;
2525	pVCpu->cpum.GstCtx.es.Attr.u &= ~X86DESCATTR_P;
2526
2527	pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
2528	pVCpu->cpum.GstCtx.cs.Attr.u &= ~X86DESCATTR_P;
2529
2530	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
2531	pVCpu->cpum.GstCtx.ss.Attr.u &= ~X86DESCATTR_P;
2532
2533	pVCpu->cpum.GstCtx.ds.Sel = uNewDS;
2534	pVCpu->cpum.GstCtx.ds.Attr.u &= ~X86DESCATTR_P;
2535
2536	pVCpu->cpum.GstCtx.fs.Sel = uNewFS;
2537	pVCpu->cpum.GstCtx.fs.Attr.u &= ~X86DESCATTR_P;
2538
2539	pVCpu->cpum.GstCtx.gs.Sel = uNewGS;
2540	pVCpu->cpum.GstCtx.gs.Attr.u &= ~X86DESCATTR_P;
2541	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2542
2543	pVCpu->cpum.GstCtx.ldtr.Sel = uNewLdt;
2544	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_STALE;
2545	pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P;
2546	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR);
2547
2548	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2549	{
2550	pVCpu->cpum.GstCtx.es.Attr.u \|= X86DESCATTR_UNUSABLE;
2551	pVCpu->cpum.GstCtx.cs.Attr.u \|= X86DESCATTR_UNUSABLE;
2552	pVCpu->cpum.GstCtx.ss.Attr.u \|= X86DESCATTR_UNUSABLE;
2553	pVCpu->cpum.GstCtx.ds.Attr.u \|= X86DESCATTR_UNUSABLE;
2554	pVCpu->cpum.GstCtx.fs.Attr.u \|= X86DESCATTR_UNUSABLE;
2555	pVCpu->cpum.GstCtx.gs.Attr.u \|= X86DESCATTR_UNUSABLE;
2556	pVCpu->cpum.GstCtx.ldtr.Attr.u \|= X86DESCATTR_UNUSABLE;
2557	}
2558
2559	/*
2560	* Switch CR3 for the new task.
2561	*/
2562	if ( fIsNewTSS386
2563	&& (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG))
2564	{
2565	/** @todo Should we update and flush TLBs only if CR3 value actually changes? */
2566	int rc = CPUMSetGuestCR3(pVCpu, uNewCr3);
2567	AssertRCSuccessReturn(rc, rc);
2568
2569	/* Inform PGM. */
2570	/** @todo Should we raise \#GP(0) here when PAE PDPEs are invalid? */
2571	rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
2572	AssertRCReturn(rc, rc);
2573	/* ignore informational status codes */
2574
2575	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3);
2576	}
2577
2578	/*
2579	* Switch LDTR for the new task.
2580	*/
2581	if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
2582	iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt);
2583	else
2584	{
2585	Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present); /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
2586
2587	IEMSELDESC DescNewLdt;
2588	rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
2589	if (rcStrict != VINF_SUCCESS)
2590	{
2591	Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
2592	uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
2593	return rcStrict;
2594	}
2595	if ( !DescNewLdt.Legacy.Gen.u1Present
2596	\|\| DescNewLdt.Legacy.Gen.u1DescType
2597	\|\| DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
2598	{
2599	Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
2600	uNewLdt, DescNewLdt.Legacy.u));
2601	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
2602	}
2603
2604	pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
2605	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
2606	pVCpu->cpum.GstCtx.ldtr.u64Base = X86DESC_BASE(&DescNewLdt.Legacy);
2607	pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
2608	pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
2609	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2610	pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
2611	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
2612	}
2613
2614	IEMSELDESC DescSS;
2615	if (IEM_IS_V86_MODE(pVCpu))
2616	{
2617	pVCpu->iem.s.uCpl = 3;
2618	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES);
2619	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS);
2620	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS);
2621	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS);
2622	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS);
2623	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS);
2624
2625	/* Quick fix: fake DescSS. / /* @todo fix the code further down? */
2626	DescSS.Legacy.u = 0;
2627	DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit;
2628	DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16;
2629	DescSS.Legacy.Gen.u16BaseLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base;
2630	DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16);
2631	DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24);
2632	DescSS.Legacy.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2633	DescSS.Legacy.Gen.u2Dpl = 3;
2634	}
2635	else
2636	{
2637	uint8_t const uNewCpl = (uNewCS & X86_SEL_RPL);
2638
2639	/*
2640	* Load the stack segment for the new task.
2641	*/
2642	if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
2643	{
2644	Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
2645	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2646	}
2647
2648	/* Fetch the descriptor. */
2649	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS);
2650	if (rcStrict != VINF_SUCCESS)
2651	{
2652	Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
2653	VBOXSTRICTRC_VAL(rcStrict)));
2654	return rcStrict;
2655	}
2656
2657	/* SS must be a data segment and writable. */
2658	if ( !DescSS.Legacy.Gen.u1DescType
2659	\|\| (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
2660	\|\| !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
2661	{
2662	Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
2663	uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
2664	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2665	}
2666
2667	/* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
2668	if ( (uNewSS & X86_SEL_RPL) != uNewCpl
2669	\|\| DescSS.Legacy.Gen.u2Dpl != uNewCpl)
2670	{
2671	Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
2672	uNewCpl));
2673	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2674	}
2675
2676	/* Is it there? */
2677	if (!DescSS.Legacy.Gen.u1Present)
2678	{
2679	Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
2680	return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2681	}
2682
2683	uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
2684	uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
2685
2686	/* Set the accessed bit before committing the result into SS. */
2687	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2688	{
2689	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
2690	if (rcStrict != VINF_SUCCESS)
2691	return rcStrict;
2692	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2693	}
2694
2695	/* Commit SS. */
2696	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
2697	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
2698	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
2699	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit;
2700	pVCpu->cpum.GstCtx.ss.u64Base = u64Base;
2701	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
2702	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
2703
2704	/* CPL has changed, update IEM before loading rest of segments. */
2705	pVCpu->iem.s.uCpl = uNewCpl;
2706
2707	/*
2708	* Load the data segments for the new task.
2709	*/
2710	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES);
2711	if (rcStrict != VINF_SUCCESS)
2712	return rcStrict;
2713	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS);
2714	if (rcStrict != VINF_SUCCESS)
2715	return rcStrict;
2716	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS);
2717	if (rcStrict != VINF_SUCCESS)
2718	return rcStrict;
2719	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS);
2720	if (rcStrict != VINF_SUCCESS)
2721	return rcStrict;
2722
2723	/*
2724	* Load the code segment for the new task.
2725	*/
2726	if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
2727	{
2728	Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
2729	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2730	}
2731
2732	/* Fetch the descriptor. */
2733	IEMSELDESC DescCS;
2734	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS);
2735	if (rcStrict != VINF_SUCCESS)
2736	{
2737	Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
2738	return rcStrict;
2739	}
2740
2741	/* CS must be a code segment. */
2742	if ( !DescCS.Legacy.Gen.u1DescType
2743	\|\| !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
2744	{
2745	Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
2746	DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
2747	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2748	}
2749
2750	/* For conforming CS, DPL must be less than or equal to the RPL. */
2751	if ( (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2752	&& DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
2753	{
2754	Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
2755	DescCS.Legacy.Gen.u2Dpl));
2756	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2757	}
2758
2759	/* For non-conforming CS, DPL must match RPL. */
2760	if ( !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2761	&& DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
2762	{
2763	Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
2764	DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
2765	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2766	}
2767
2768	/* Is it there? */
2769	if (!DescCS.Legacy.Gen.u1Present)
2770	{
2771	Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
2772	return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2773	}
2774
2775	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
2776	u64Base = X86DESC_BASE(&DescCS.Legacy);
2777
2778	/* Set the accessed bit before committing the result into CS. */
2779	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2780	{
2781	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
2782	if (rcStrict != VINF_SUCCESS)
2783	return rcStrict;
2784	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2785	}
2786
2787	/* Commit CS. */
2788	pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
2789	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS;
2790	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
2791	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
2792	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
2793	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2794	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
2795	}
2796
2797	/** @todo Debug trap. */
2798	if (fIsNewTSS386 && fNewDebugTrap)
2799	Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
2800
2801	/*
2802	* Construct the error code masks based on what caused this task switch.
2803	* See Intel Instruction reference for INT.
2804	*/
2805	uint16_t uExt;
2806	if ( enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
2807	&& ( !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
2808	\|\| (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)))
2809	{
2810	uExt = 1;
2811	}
2812	else
2813	uExt = 0;
2814
2815	/*
2816	* Push any error code on to the new stack.
2817	*/
2818	if (fFlags & IEM_XCPT_FLAGS_ERR)
2819	{
2820	Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
2821	uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
2822	uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2;
2823
2824	/* Check that there is sufficient space on the stack. */
2825	/** @todo Factor out segment limit checking for normal/expand down segments
2826	* into a separate function. */
2827	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
2828	{
2829	if ( pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS
2830	\|\| pVCpu->cpum.GstCtx.esp < cbStackFrame)
2831	{
2832	/** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
2833	Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n",
2834	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
2835	return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
2836	}
2837	}
2838	else
2839	{
2840	if ( pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2841	\|\| pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1))
2842	{
2843	Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n",
2844	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
2845	return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
2846	}
2847	}
2848
2849
2850	if (fIsNewTSS386)
2851	rcStrict = iemMemStackPushU32(pVCpu, uErr);
2852	else
2853	rcStrict = iemMemStackPushU16(pVCpu, uErr);
2854	if (rcStrict != VINF_SUCCESS)
2855	{
2856	Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n",
2857	fIsNewTSS386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict)));
2858	return rcStrict;
2859	}
2860	}
2861
2862	/* Check the new EIP against the new CS limit. */
2863	if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit)
2864	{
2865	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n",
2866	pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit));
2867	/** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
2868	return iemRaiseGeneralProtectionFault(pVCpu, uExt);
2869	}
2870
2871	Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
2872	pVCpu->cpum.GstCtx.ss.Sel));
2873	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
2874	}
2875
2876
2877	/**
2878	* Implements exceptions and interrupts for protected mode.
2879	*
2880	* @returns VBox strict status code.
2881	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2882	* @param cbInstr The number of bytes to offset rIP by in the return
2883	* address.
2884	* @param u8Vector The interrupt / exception vector number.
2885	* @param fFlags The flags.
2886	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2887	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2888	*/
2889	static VBOXSTRICTRC
2890	iemRaiseXcptOrIntInProtMode(PVMCPUCC pVCpu,
2891	uint8_t cbInstr,
2892	uint8_t u8Vector,
2893	uint32_t fFlags,
2894	uint16_t uErr,
2895	uint64_t uCr2) RT_NOEXCEPT
2896	{
2897	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
2898
2899	/*
2900	* Read the IDT entry.
2901	*/
2902	if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
2903	{
2904	Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
2905	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2906	}
2907	X86DESC Idte;
2908	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX,
2909	pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector);
2910	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
2911	{
2912	Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
2913	return rcStrict;
2914	}
2915	Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
2916	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
2917	Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
2918
2919	/*
2920	* Check the descriptor type, DPL and such.
2921	* ASSUMES this is done in the same order as described for call-gate calls.
2922	*/
2923	if (Idte.Gate.u1DescType)
2924	{
2925	Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
2926	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2927	}
2928	bool fTaskGate = false;
2929	uint8_t f32BitGate = true;
2930	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
2931	switch (Idte.Gate.u4Type)
2932	{
2933	case X86_SEL_TYPE_SYS_UNDEFINED:
2934	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
2935	case X86_SEL_TYPE_SYS_LDT:
2936	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
2937	case X86_SEL_TYPE_SYS_286_CALL_GATE:
2938	case X86_SEL_TYPE_SYS_UNDEFINED2:
2939	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
2940	case X86_SEL_TYPE_SYS_UNDEFINED3:
2941	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
2942	case X86_SEL_TYPE_SYS_386_CALL_GATE:
2943	case X86_SEL_TYPE_SYS_UNDEFINED4:
2944	{
2945	/** @todo check what actually happens when the type is wrong...
2946	* esp. call gates. */
2947	Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
2948	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2949	}
2950
2951	case X86_SEL_TYPE_SYS_286_INT_GATE:
2952	f32BitGate = false;
2953	RT_FALL_THRU();
2954	case X86_SEL_TYPE_SYS_386_INT_GATE:
2955	fEflToClear \|= X86_EFL_IF;
2956	break;
2957
2958	case X86_SEL_TYPE_SYS_TASK_GATE:
2959	fTaskGate = true;
2960	#ifndef IEM_IMPLEMENTS_TASKSWITCH
2961	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
2962	#endif
2963	break;
2964
2965	case X86_SEL_TYPE_SYS_286_TRAP_GATE:
2966	f32BitGate = false;
2967	case X86_SEL_TYPE_SYS_386_TRAP_GATE:
2968	break;
2969
2970	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2971	}
2972
2973	/* Check DPL against CPL if applicable. */
2974	if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
2975	{
2976	if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
2977	{
2978	Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
2979	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2980	}
2981	}
2982
2983	/* Is it there? */
2984	if (!Idte.Gate.u1Present)
2985	{
2986	Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
2987	return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2988	}
2989
2990	/* Is it a task-gate? */
2991	if (fTaskGate)
2992	{
2993	/*
2994	* Construct the error code masks based on what caused this task switch.
2995	* See Intel Instruction reference for INT.
2996	*/
2997	uint16_t const uExt = ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
2998	&& !(fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)) ? 0 : 1;
2999	uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
3000	RTSEL SelTSS = Idte.Gate.u16Sel;
3001
3002	/*
3003	* Fetch the TSS descriptor in the GDT.
3004	*/
3005	IEMSELDESC DescTSS;
3006	rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) \| uExt);
3007	if (rcStrict != VINF_SUCCESS)
3008	{
3009	Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS,
3010	VBOXSTRICTRC_VAL(rcStrict)));
3011	return rcStrict;
3012	}
3013
3014	/* The TSS descriptor must be a system segment and be available (not busy). */
3015	if ( DescTSS.Legacy.Gen.u1DescType
3016	\|\| ( DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
3017	&& DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
3018	{
3019	Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
3020	u8Vector, SelTSS, DescTSS.Legacy.au64));
3021	return iemRaiseGeneralProtectionFault(pVCpu, (SelTSS & uSelMask) \| uExt);
3022	}
3023
3024	/* The TSS must be present. */
3025	if (!DescTSS.Legacy.Gen.u1Present)
3026	{
3027	Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64));
3028	return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTSS & uSelMask) \| uExt);
3029	}
3030
3031	/* Do the actual task switch. */
3032	return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT,
3033	(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip,
3034	fFlags, uErr, uCr2, SelTSS, &DescTSS);
3035	}
3036
3037	/* A null CS is bad. */
3038	RTSEL NewCS = Idte.Gate.u16Sel;
3039	if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3040	{
3041	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3042	return iemRaiseGeneralProtectionFault0(pVCpu);
3043	}
3044
3045	/* Fetch the descriptor for the new CS. */
3046	IEMSELDESC DescCS;
3047	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
3048	if (rcStrict != VINF_SUCCESS)
3049	{
3050	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3051	return rcStrict;
3052	}
3053
3054	/* Must be a code segment. */
3055	if (!DescCS.Legacy.Gen.u1DescType)
3056	{
3057	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3058	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3059	}
3060	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3061	{
3062	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3063	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3064	}
3065
3066	/* Don't allow lowering the privilege level. */
3067	/** @todo Does the lowering of privileges apply to software interrupts
3068	* only? This has bearings on the more-privileged or
3069	* same-privilege stack behavior further down. A testcase would
3070	* be nice. */
3071	if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
3072	{
3073	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3074	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3075	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3076	}
3077
3078	/* Make sure the selector is present. */
3079	if (!DescCS.Legacy.Gen.u1Present)
3080	{
3081	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3082	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
3083	}
3084
3085	/* Check the new EIP against the new CS limit. */
3086	uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
3087	\|\| Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
3088	? Idte.Gate.u16OffsetLow
3089	: Idte.Gate.u16OffsetLow \| ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
3090	uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
3091	if (uNewEip > cbLimitCS)
3092	{
3093	Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
3094	u8Vector, uNewEip, cbLimitCS, NewCS));
3095	return iemRaiseGeneralProtectionFault(pVCpu, 0);
3096	}
3097	Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS));
3098
3099	/* Calc the flag image to push. */
3100	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3101	if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP \| IEM_XCPT_FLAGS_T_SOFT_INT))
3102	fEfl &= ~X86_EFL_RF;
3103	else
3104	fEfl \|= X86_EFL_RF; /* Vagueness is all I've found on this so far... / /* @todo Automatically pushing EFLAGS.RF. */
3105
3106	/* From V8086 mode only go to CPL 0. */
3107	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3108	? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
3109	if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
3110	{
3111	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
3112	return iemRaiseGeneralProtectionFault(pVCpu, 0);
3113	}
3114
3115	/*
3116	* If the privilege level changes, we need to get a new stack from the TSS.
3117	* This in turns means validating the new SS and ESP...
3118	*/
3119	if (uNewCpl != pVCpu->iem.s.uCpl)
3120	{
3121	RTSEL NewSS;
3122	uint32_t uNewEsp;
3123	rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp);
3124	if (rcStrict != VINF_SUCCESS)
3125	return rcStrict;
3126
3127	IEMSELDESC DescSS;
3128	rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS);
3129	if (rcStrict != VINF_SUCCESS)
3130	return rcStrict;
3131	/* If the new SS is 16-bit, we are only going to use SP, not ESP. */
3132	if (!DescSS.Legacy.Gen.u1DefBig)
3133	{
3134	Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp));
3135	uNewEsp = (uint16_t)uNewEsp;
3136	}
3137
3138	Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
3139
3140	/* Check that there is sufficient space for the stack frame. */
3141	uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
3142	uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
3143	? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
3144	: (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
3145
3146	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
3147	{
3148	if ( uNewEsp - 1 > cbLimitSS
3149	\|\| uNewEsp < cbStackFrame)
3150	{
3151	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
3152	u8Vector, NewSS, uNewEsp, cbStackFrame));
3153	return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
3154	}
3155	}
3156	else
3157	{
3158	if ( uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX)
3159	\|\| uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1))
3160	{
3161	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n",
3162	u8Vector, NewSS, uNewEsp, cbStackFrame));
3163	return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
3164	}
3165	}
3166
3167	/*
3168	* Start making changes.
3169	*/
3170
3171	/* Set the new CPL so that stack accesses use it. */
3172	uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
3173	pVCpu->iem.s.uCpl = uNewCpl;
3174
3175	/* Create the stack frame. */
3176	RTPTRUNION uStackFrame;
3177	rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
3178	uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy),
3179	IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
3180	if (rcStrict != VINF_SUCCESS)
3181	return rcStrict;
3182	void * const pvStackFrame = uStackFrame.pv;
3183	if (f32BitGate)
3184	{
3185	if (fFlags & IEM_XCPT_FLAGS_ERR)
3186	*uStackFrame.pu32++ = uErr;
3187	uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3188	uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl;
3189	uStackFrame.pu32[2] = fEfl;
3190	uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp;
3191	uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel;
3192	Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
3193	if (fEfl & X86_EFL_VM)
3194	{
3195	uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel;
3196	uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel;
3197	uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel;
3198	uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel;
3199	uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel;
3200	}
3201	}
3202	else
3203	{
3204	if (fFlags & IEM_XCPT_FLAGS_ERR)
3205	*uStackFrame.pu16++ = uErr;
3206	uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
3207	uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl;
3208	uStackFrame.pu16[2] = fEfl;
3209	uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp;
3210	uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel;
3211	Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp));
3212	if (fEfl & X86_EFL_VM)
3213	{
3214	uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
3215	uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel;
3216	uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel;
3217	uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel;
3218	uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel;
3219	}
3220	}
3221	rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
3222	if (rcStrict != VINF_SUCCESS)
3223	return rcStrict;
3224
3225	/* Mark the selectors 'accessed' (hope this is the correct time). */
3226	/** @todo testcase: excatly _when_ are the accessed bits set - before or
3227	* after pushing the stack frame? (Write protect the gdt + stack to
3228	* find out.) */
3229	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3230	{
3231	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3232	if (rcStrict != VINF_SUCCESS)
3233	return rcStrict;
3234	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3235	}
3236
3237	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3238	{
3239	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS);
3240	if (rcStrict != VINF_SUCCESS)
3241	return rcStrict;
3242	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3243	}
3244
3245	/*
3246	* Start comitting the register changes (joins with the DPL=CPL branch).
3247	*/
3248	pVCpu->cpum.GstCtx.ss.Sel = NewSS;
3249	pVCpu->cpum.GstCtx.ss.ValidSel = NewSS;
3250	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3251	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSS;
3252	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
3253	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3254	/** @todo When coming from 32-bit code and operating with a 16-bit TSS and
3255	* 16-bit handler, the high word of ESP remains unchanged (i.e. only
3256	* SP is loaded).
3257	* Need to check the other combinations too:
3258	* - 16-bit TSS, 32-bit handler
3259	* - 32-bit TSS, 16-bit handler */
3260	if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3261	pVCpu->cpum.GstCtx.sp = (uint16_t)(uNewEsp - cbStackFrame);
3262	else
3263	pVCpu->cpum.GstCtx.rsp = uNewEsp - cbStackFrame;
3264
3265	if (fEfl & X86_EFL_VM)
3266	{
3267	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs);
3268	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs);
3269	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es);
3270	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds);
3271	}
3272	}
3273	/*
3274	* Same privilege, no stack change and smaller stack frame.
3275	*/
3276	else
3277	{
3278	uint64_t uNewRsp;
3279	RTPTRUNION uStackFrame;
3280	uint8_t const cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
3281	rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, f32BitGate ? 3 : 1, &uStackFrame.pv, &uNewRsp);
3282	if (rcStrict != VINF_SUCCESS)
3283	return rcStrict;
3284	void * const pvStackFrame = uStackFrame.pv;
3285
3286	if (f32BitGate)
3287	{
3288	if (fFlags & IEM_XCPT_FLAGS_ERR)
3289	*uStackFrame.pu32++ = uErr;
3290	uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3291	uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| pVCpu->iem.s.uCpl;
3292	uStackFrame.pu32[2] = fEfl;
3293	}
3294	else
3295	{
3296	if (fFlags & IEM_XCPT_FLAGS_ERR)
3297	*uStackFrame.pu16++ = uErr;
3298	uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3299	uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| pVCpu->iem.s.uCpl;
3300	uStackFrame.pu16[2] = fEfl;
3301	}
3302	rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
3303	if (rcStrict != VINF_SUCCESS)
3304	return rcStrict;
3305
3306	/* Mark the CS selector as 'accessed'. */
3307	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3308	{
3309	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3310	if (rcStrict != VINF_SUCCESS)
3311	return rcStrict;
3312	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3313	}
3314
3315	/*
3316	* Start committing the register changes (joins with the other branch).
3317	*/
3318	pVCpu->cpum.GstCtx.rsp = uNewRsp;
3319	}
3320
3321	/* ... register committing continues. */
3322	pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3323	pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3324	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3325	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3326	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3327	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3328
3329	pVCpu->cpum.GstCtx.rip = uNewEip; /* (The entire register is modified, see pe16_32 bs3kit tests.) */
3330	fEfl &= ~fEflToClear;
3331	IEMMISC_SET_EFL(pVCpu, fEfl);
3332
3333	if (fFlags & IEM_XCPT_FLAGS_CR2)
3334	pVCpu->cpum.GstCtx.cr2 = uCr2;
3335
3336	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3337	iemRaiseXcptAdjustState(pVCpu, u8Vector);
3338
3339	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3340	}
3341
3342
3343	/**
3344	* Implements exceptions and interrupts for long mode.
3345	*
3346	* @returns VBox strict status code.
3347	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3348	* @param cbInstr The number of bytes to offset rIP by in the return
3349	* address.
3350	* @param u8Vector The interrupt / exception vector number.
3351	* @param fFlags The flags.
3352	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3353	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3354	*/
3355	static VBOXSTRICTRC
3356	iemRaiseXcptOrIntInLongMode(PVMCPUCC pVCpu,
3357	uint8_t cbInstr,
3358	uint8_t u8Vector,
3359	uint32_t fFlags,
3360	uint16_t uErr,
3361	uint64_t uCr2) RT_NOEXCEPT
3362	{
3363	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3364
3365	/*
3366	* Read the IDT entry.
3367	*/
3368	uint16_t offIdt = (uint16_t)u8Vector << 4;
3369	if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7)
3370	{
3371	Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
3372	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3373	}
3374	X86DESC64 Idte;
3375	#ifdef _MSC_VER /* Shut up silly compiler warning. */
3376	Idte.au64[0] = 0;
3377	Idte.au64[1] = 0;
3378	#endif
3379	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt);
3380	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3381	rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8);
3382	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3383	{
3384	Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
3385	return rcStrict;
3386	}
3387	Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
3388	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
3389	Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
3390
3391	/*
3392	* Check the descriptor type, DPL and such.
3393	* ASSUMES this is done in the same order as described for call-gate calls.
3394	*/
3395	if (Idte.Gate.u1DescType)
3396	{
3397	Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3398	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3399	}
3400	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
3401	switch (Idte.Gate.u4Type)
3402	{
3403	case AMD64_SEL_TYPE_SYS_INT_GATE:
3404	fEflToClear \|= X86_EFL_IF;
3405	break;
3406	case AMD64_SEL_TYPE_SYS_TRAP_GATE:
3407	break;
3408
3409	default:
3410	Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3411	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3412	}
3413
3414	/* Check DPL against CPL if applicable. */
3415	if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
3416	{
3417	if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
3418	{
3419	Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
3420	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3421	}
3422	}
3423
3424	/* Is it there? */
3425	if (!Idte.Gate.u1Present)
3426	{
3427	Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
3428	return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3429	}
3430
3431	/* A null CS is bad. */
3432	RTSEL NewCS = Idte.Gate.u16Sel;
3433	if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3434	{
3435	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3436	return iemRaiseGeneralProtectionFault0(pVCpu);
3437	}
3438
3439	/* Fetch the descriptor for the new CS. */
3440	IEMSELDESC DescCS;
3441	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP);
3442	if (rcStrict != VINF_SUCCESS)
3443	{
3444	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3445	return rcStrict;
3446	}
3447
3448	/* Must be a 64-bit code segment. */
3449	if (!DescCS.Long.Gen.u1DescType)
3450	{
3451	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3452	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3453	}
3454	if ( !DescCS.Long.Gen.u1Long
3455	\|\| DescCS.Long.Gen.u1DefBig
3456	\|\| !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
3457	{
3458	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
3459	u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
3460	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3461	}
3462
3463	/* Don't allow lowering the privilege level. For non-conforming CS
3464	selectors, the CS.DPL sets the privilege level the trap/interrupt
3465	handler runs at. For conforming CS selectors, the CPL remains
3466	unchanged, but the CS.DPL must be <= CPL. */
3467	/** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
3468	* when CPU in Ring-0. Result \#GP? */
3469	if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
3470	{
3471	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3472	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3473	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3474	}
3475
3476
3477	/* Make sure the selector is present. */
3478	if (!DescCS.Legacy.Gen.u1Present)
3479	{
3480	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3481	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
3482	}
3483
3484	/* Check that the new RIP is canonical. */
3485	uint64_t const uNewRip = Idte.Gate.u16OffsetLow
3486	\| ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
3487	\| ((uint64_t)Idte.Gate.u32OffsetTop << 32);
3488	if (!IEM_IS_CANONICAL(uNewRip))
3489	{
3490	Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
3491	return iemRaiseGeneralProtectionFault0(pVCpu);
3492	}
3493
3494	/*
3495	* If the privilege level changes or if the IST isn't zero, we need to get
3496	* a new stack from the TSS.
3497	*/
3498	uint64_t uNewRsp;
3499	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3500	? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
3501	if ( uNewCpl != pVCpu->iem.s.uCpl
3502	\|\| Idte.Gate.u3IST != 0)
3503	{
3504	rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
3505	if (rcStrict != VINF_SUCCESS)
3506	return rcStrict;
3507	}
3508	else
3509	uNewRsp = pVCpu->cpum.GstCtx.rsp;
3510	uNewRsp &= ~(uint64_t)0xf;
3511
3512	/*
3513	* Calc the flag image to push.
3514	*/
3515	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3516	if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP \| IEM_XCPT_FLAGS_T_SOFT_INT))
3517	fEfl &= ~X86_EFL_RF;
3518	else
3519	fEfl \|= X86_EFL_RF; /* Vagueness is all I've found on this so far... / /* @todo Automatically pushing EFLAGS.RF. */
3520
3521	/*
3522	* Start making changes.
3523	*/
3524	/* Set the new CPL so that stack accesses use it. */
3525	uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
3526	pVCpu->iem.s.uCpl = uNewCpl;
3527
3528	/* Create the stack frame. */
3529	uint32_t cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
3530	RTPTRUNION uStackFrame;
3531	rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
3532	uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
3533	if (rcStrict != VINF_SUCCESS)
3534	return rcStrict;
3535	void * const pvStackFrame = uStackFrame.pv;
3536
3537	if (fFlags & IEM_XCPT_FLAGS_ERR)
3538	*uStackFrame.pu64++ = uErr;
3539	uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip;
3540	uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl; /* CPL paranoia */
3541	uStackFrame.pu64[2] = fEfl;
3542	uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp;
3543	uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel;
3544	rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
3545	if (rcStrict != VINF_SUCCESS)
3546	return rcStrict;
3547
3548	/* Mark the CS selectors 'accessed' (hope this is the correct time). */
3549	/** @todo testcase: excatly _when_ are the accessed bits set - before or
3550	* after pushing the stack frame? (Write protect the gdt + stack to
3551	* find out.) */
3552	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3553	{
3554	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3555	if (rcStrict != VINF_SUCCESS)
3556	return rcStrict;
3557	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3558	}
3559
3560	/*
3561	* Start comitting the register changes.
3562	*/
3563	/** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
3564	* hidden registers when interrupting 32-bit or 16-bit code! */
3565	if (uNewCpl != uOldCpl)
3566	{
3567	pVCpu->cpum.GstCtx.ss.Sel = 0 \| uNewCpl;
3568	pVCpu->cpum.GstCtx.ss.ValidSel = 0 \| uNewCpl;
3569	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3570	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
3571	pVCpu->cpum.GstCtx.ss.u64Base = 0;
3572	pVCpu->cpum.GstCtx.ss.Attr.u = (uNewCpl << X86DESCATTR_DPL_SHIFT) \| X86DESCATTR_UNUSABLE;
3573	}
3574	pVCpu->cpum.GstCtx.rsp = uNewRsp - cbStackFrame;
3575	pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3576	pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3577	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3578	pVCpu->cpum.GstCtx.cs.u32Limit = X86DESC_LIMIT_G(&DescCS.Legacy);
3579	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3580	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3581	pVCpu->cpum.GstCtx.rip = uNewRip;
3582
3583	fEfl &= ~fEflToClear;
3584	IEMMISC_SET_EFL(pVCpu, fEfl);
3585
3586	if (fFlags & IEM_XCPT_FLAGS_CR2)
3587	pVCpu->cpum.GstCtx.cr2 = uCr2;
3588
3589	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3590	iemRaiseXcptAdjustState(pVCpu, u8Vector);
3591
3592	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3593	}
3594
3595
3596	/**
3597	* Implements exceptions and interrupts.
3598	*
3599	* All exceptions and interrupts goes thru this function!
3600	*
3601	* @returns VBox strict status code.
3602	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3603	* @param cbInstr The number of bytes to offset rIP by in the return
3604	* address.
3605	* @param u8Vector The interrupt / exception vector number.
3606	* @param fFlags The flags.
3607	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3608	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3609	*/
3610	VBOXSTRICTRC
3611	iemRaiseXcptOrInt(PVMCPUCC pVCpu,
3612	uint8_t cbInstr,
3613	uint8_t u8Vector,
3614	uint32_t fFlags,
3615	uint16_t uErr,
3616	uint64_t uCr2) RT_NOEXCEPT
3617	{
3618	/*
3619	* Get all the state that we might need here.
3620	*/
3621	IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3622	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3623
3624	#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */
3625	/*
3626	* Flush prefetch buffer
3627	*/
3628	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
3629	#endif
3630
3631	/*
3632	* Perform the V8086 IOPL check and upgrade the fault without nesting.
3633	*/
3634	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1VM
3635	&& pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3
3636	&& (fFlags & ( IEM_XCPT_FLAGS_T_SOFT_INT
3637	\| IEM_XCPT_FLAGS_BP_INSTR
3638	\| IEM_XCPT_FLAGS_ICEBP_INSTR
3639	\| IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
3640	&& (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
3641	{
3642	Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
3643	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR;
3644	u8Vector = X86_XCPT_GP;
3645	uErr = 0;
3646	}
3647	#ifdef DBGFTRACE_ENABLED
3648	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
3649	pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
3650	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp);
3651	#endif
3652
3653	/*
3654	* Evaluate whether NMI blocking should be in effect.
3655	* Normally, NMI blocking is in effect whenever we inject an NMI.
3656	*/
3657	bool fBlockNmi;
3658	if ( u8Vector == X86_XCPT_NMI
3659	&& (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
3660	fBlockNmi = true;
3661	else
3662	fBlockNmi = false;
3663
3664	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3665	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3666	{
3667	VBOXSTRICTRC rcStrict0 = iemVmxVmexitEvent(pVCpu, u8Vector, fFlags, uErr, uCr2, cbInstr);
3668	if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
3669	return rcStrict0;
3670
3671	/* If virtual-NMI blocking is in effect for the nested-guest, guest NMIs are not blocked. */
3672	if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
3673	{
3674	Assert(CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_VIRT_NMI));
3675	fBlockNmi = false;
3676	}
3677	}
3678	#endif
3679
3680	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3681	if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
3682	{
3683	/*
3684	* If the event is being injected as part of VMRUN, it isn't subject to event
3685	* intercepts in the nested-guest. However, secondary exceptions that occur
3686	* during injection of any event -are- subject to exception intercepts.
3687	*
3688	* See AMD spec. 15.20 "Event Injection".
3689	*/
3690	if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents)
3691	pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = true;
3692	else
3693	{
3694	/*
3695	* Check and handle if the event being raised is intercepted.
3696	*/
3697	VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, u8Vector, fFlags, uErr, uCr2);
3698	if (rcStrict0 != VINF_SVM_INTERCEPT_NOT_ACTIVE)
3699	return rcStrict0;
3700	}
3701	}
3702	#endif
3703
3704	/*
3705	* Set NMI blocking if necessary.
3706	*/
3707	if ( fBlockNmi
3708	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3709	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3710
3711	/*
3712	* Do recursion accounting.
3713	*/
3714	uint8_t const uPrevXcpt = pVCpu->iem.s.uCurXcpt;
3715	uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt;
3716	if (pVCpu->iem.s.cXcptRecursions == 0)
3717	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
3718	u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2));
3719	else
3720	{
3721	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
3722	u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt,
3723	pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt));
3724
3725	if (pVCpu->iem.s.cXcptRecursions >= 4)
3726	{
3727	#ifdef DEBUG_bird
3728	AssertFailed();
3729	#endif
3730	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
3731	}
3732
3733	/*
3734	* Evaluate the sequence of recurring events.
3735	*/
3736	IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector,
3737	NULL /* pXcptRaiseInfo */);
3738	if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT)
3739	{ /* likely */ }
3740	else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT)
3741	{
3742	Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt));
3743	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR;
3744	u8Vector = X86_XCPT_DF;
3745	uErr = 0;
3746	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3747	/* VMX nested-guest #DF intercept needs to be checked here. */
3748	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3749	{
3750	VBOXSTRICTRC rcStrict0 = iemVmxVmexitEventDoubleFault(pVCpu);
3751	if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
3752	return rcStrict0;
3753	}
3754	#endif
3755	/* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */
3756	if (IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF))
3757	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
3758	}
3759	else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT)
3760	{
3761	Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt));
3762	return iemInitiateCpuShutdown(pVCpu);
3763	}
3764	else if (enmRaise == IEMXCPTRAISE_CPU_HANG)
3765	{
3766	/* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */
3767	Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n"));
3768	if ( !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
3769	&& !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
3770	return VERR_EM_GUEST_CPU_HANG;
3771	}
3772	else
3773	{
3774	AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n",
3775	enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags));
3776	return VERR_IEM_IPE_9;
3777	}
3778
3779	/*
3780	* The 'EXT' bit is set when an exception occurs during deliver of an external
3781	* event (such as an interrupt or earlier exception)[1]. Privileged software
3782	* exception (INT1) also sets the EXT bit[2]. Exceptions generated by software
3783	* interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set.
3784	*
3785	* [1] - Intel spec. 6.13 "Error Code"
3786	* [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection".
3787	* [3] - Intel Instruction reference for INT n.
3788	*/
3789	if ( (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_T_EXT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR))
3790	&& (fFlags & IEM_XCPT_FLAGS_ERR)
3791	&& u8Vector != X86_XCPT_PF
3792	&& u8Vector != X86_XCPT_DF)
3793	{
3794	uErr \|= X86_TRAP_ERR_EXTERNAL;
3795	}
3796	}
3797
3798	pVCpu->iem.s.cXcptRecursions++;
3799	pVCpu->iem.s.uCurXcpt = u8Vector;
3800	pVCpu->iem.s.fCurXcpt = fFlags;
3801	pVCpu->iem.s.uCurXcptErr = uErr;
3802	pVCpu->iem.s.uCurXcptCr2 = uCr2;
3803
3804	/*
3805	* Extensive logging.
3806	*/
3807	#if defined(LOG_ENABLED) && defined(IN_RING3)
3808	if (LogIs3Enabled())
3809	{
3810	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK);
3811	PVM pVM = pVCpu->CTX_SUFF(pVM);
3812	char szRegs[4096];
3813	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
3814	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
3815	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
3816	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
3817	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
3818	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
3819	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
3820	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
3821	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
3822	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
3823	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
3824	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
3825	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
3826	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
3827	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
3828	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
3829	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
3830	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
3831	" efer=%016VR{efer}\n"
3832	" pat=%016VR{pat}\n"
3833	" sf_mask=%016VR{sf_mask}\n"
3834	"krnl_gs_base=%016VR{krnl_gs_base}\n"
3835	" lstar=%016VR{lstar}\n"
3836	" star=%016VR{star} cstar=%016VR{cstar}\n"
3837	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
3838	);
3839
3840	char szInstr[256];
3841	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
3842	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
3843	szInstr, sizeof(szInstr), NULL);
3844	Log3(("%s%s\n", szRegs, szInstr));
3845	}
3846	#endif /* LOG_ENABLED */
3847
3848	/*
3849	* Stats.
3850	*/
3851	if (!(fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
3852	STAM_REL_STATS({ pVCpu->iem.s.aStatInts[u8Vector] += 1; });
3853	else if (u8Vector <= X86_XCPT_LAST)
3854	{
3855	STAM_REL_COUNTER_INC(&pVCpu->iem.s.aStatXcpts[u8Vector]);
3856	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_XCPT, u8Vector),
3857	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base, ASMReadTSC());
3858	}
3859
3860	/*
3861	* #PF's implies a INVLPG for the CR2 value (see 4.10.1.1 in Intel SDM Vol 3)
3862	* to ensure that a stale TLB or paging cache entry will only cause one
3863	* spurious #PF.
3864	*/
3865	if ( u8Vector == X86_XCPT_PF
3866	&& (fFlags & (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_CR2)) == (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_CR2))
3867	IEMTlbInvalidatePage(pVCpu, uCr2);
3868
3869	/*
3870	* Call the mode specific worker function.
3871	*/
3872	VBOXSTRICTRC rcStrict;
3873	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
3874	rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3875	else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
3876	rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3877	else
3878	rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3879
3880	/* Flush the prefetch buffer. */
3881	#ifdef IEM_WITH_CODE_TLB
3882	pVCpu->iem.s.pbInstrBuf = NULL;
3883	#else
3884	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
3885	#endif
3886
3887	/*
3888	* Unwind.
3889	*/
3890	pVCpu->iem.s.cXcptRecursions--;
3891	pVCpu->iem.s.uCurXcpt = uPrevXcpt;
3892	pVCpu->iem.s.fCurXcpt = fPrevXcpt;
3893	Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n",
3894	VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, pVCpu->iem.s.uCpl,
3895	pVCpu->iem.s.cXcptRecursions + 1));
3896	return rcStrict;
3897	}
3898
3899	#ifdef IEM_WITH_SETJMP
3900	/**
3901	* See iemRaiseXcptOrInt. Will not return.
3902	*/
3903	DECL_NO_RETURN(void)
3904	iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu,
3905	uint8_t cbInstr,
3906	uint8_t u8Vector,
3907	uint32_t fFlags,
3908	uint16_t uErr,
3909	uint64_t uCr2) RT_NOEXCEPT
3910	{
3911	VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3912	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
3913	}
3914	#endif
3915
3916
3917	/** \#DE - 00. */
3918	VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT
3919	{
3920	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3921	}
3922
3923
3924	/** \#DB - 01.
3925	* @note This automatically clear DR7.GD. */
3926	VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT
3927	{
3928	/** @todo set/clear RF. */
3929	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
3930	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3931	}
3932
3933
3934	/** \#BR - 05. */
3935	VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT
3936	{
3937	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3938	}
3939
3940
3941	/** \#UD - 06. */
3942	VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT
3943	{
3944	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3945	}
3946
3947
3948	/** \#NM - 07. */
3949	VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT
3950	{
3951	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3952	}
3953
3954
3955	/** \#TS(err) - 0a. */
3956	VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
3957	{
3958	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
3959	}
3960
3961
3962	/** \#TS(tr) - 0a. */
3963	VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT
3964	{
3965	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3966	pVCpu->cpum.GstCtx.tr.Sel, 0);
3967	}
3968
3969
3970	/** \#TS(0) - 0a. */
3971	VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
3972	{
3973	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3974	0, 0);
3975	}
3976
3977
3978	/** \#TS(err) - 0a. */
3979	VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
3980	{
3981	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3982	uSel & X86_SEL_MASK_OFF_RPL, 0);
3983	}
3984
3985
3986	/** \#NP(err) - 0b. */
3987	VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
3988	{
3989	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
3990	}
3991
3992
3993	/** \#NP(sel) - 0b. */
3994	VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
3995	{
3996	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3997	uSel & ~X86_SEL_RPL, 0);
3998	}
3999
4000
4001	/** \#SS(seg) - 0c. */
4002	VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
4003	{
4004	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4005	uSel & ~X86_SEL_RPL, 0);
4006	}
4007
4008
4009	/** \#SS(err) - 0c. */
4010	VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4011	{
4012	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4013	}
4014
4015
4016	/** \#GP(n) - 0d. */
4017	VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4018	{
4019	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4020	}
4021
4022
4023	/** \#GP(0) - 0d. */
4024	VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
4025	{
4026	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4027	}
4028
4029	#ifdef IEM_WITH_SETJMP
4030	/** \#GP(0) - 0d. */
4031	DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) RT_NOEXCEPT
4032	{
4033	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4034	}
4035	#endif
4036
4037
4038	/** \#GP(sel) - 0d. */
4039	VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4040	{
4041	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4042	Sel & ~X86_SEL_RPL, 0);
4043	}
4044
4045
4046	/** \#GP(0) - 0d. */
4047	VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT
4048	{
4049	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4050	}
4051
4052
4053	/** \#GP(sel) - 0d. */
4054	VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4055	{
4056	NOREF(iSegReg); NOREF(fAccess);
4057	return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4058	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4059	}
4060
4061	#ifdef IEM_WITH_SETJMP
4062	/** \#GP(sel) - 0d, longjmp. */
4063	DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4064	{
4065	NOREF(iSegReg); NOREF(fAccess);
4066	iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4067	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4068	}
4069	#endif
4070
4071	/** \#GP(sel) - 0d. */
4072	VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4073	{
4074	NOREF(Sel);
4075	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4076	}
4077
4078	#ifdef IEM_WITH_SETJMP
4079	/** \#GP(sel) - 0d, longjmp. */
4080	DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4081	{
4082	NOREF(Sel);
4083	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4084	}
4085	#endif
4086
4087
4088	/** \#GP(sel) - 0d. */
4089	VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4090	{
4091	NOREF(iSegReg); NOREF(fAccess);
4092	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4093	}
4094
4095	#ifdef IEM_WITH_SETJMP
4096	/** \#GP(sel) - 0d, longjmp. */
4097	DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4098	{
4099	NOREF(iSegReg); NOREF(fAccess);
4100	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4101	}
4102	#endif
4103
4104
4105	/** \#PF(n) - 0e. */
4106	VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT
4107	{
4108	uint16_t uErr;
4109	switch (rc)
4110	{
4111	case VERR_PAGE_NOT_PRESENT:
4112	case VERR_PAGE_TABLE_NOT_PRESENT:
4113	case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
4114	case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
4115	uErr = 0;
4116	break;
4117
4118	default:
4119	AssertMsgFailed(("%Rrc\n", rc));
4120	RT_FALL_THRU();
4121	case VERR_ACCESS_DENIED:
4122	uErr = X86_TRAP_PF_P;
4123	break;
4124
4125	/** @todo reserved */
4126	}
4127
4128	if (pVCpu->iem.s.uCpl == 3)
4129	uErr \|= X86_TRAP_PF_US;
4130
4131	if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
4132	&& ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
4133	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) )
4134	uErr \|= X86_TRAP_PF_ID;
4135
4136	#if 0 /* This is so much non-sense, really. Why was it done like that? */
4137	/* Note! RW access callers reporting a WRITE protection fault, will clear
4138	the READ flag before calling. So, read-modify-write accesses (RW)
4139	can safely be reported as READ faults. */
4140	if ((fAccess & (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
4141	uErr \|= X86_TRAP_PF_RW;
4142	#else
4143	if (fAccess & IEM_ACCESS_TYPE_WRITE)
4144	{
4145	/// @todo r=bird: bs3-cpu-basic-2 wants X86_TRAP_PF_RW for xchg and cmpxchg
4146	/// (regardless of outcome of the comparison in the latter case).
4147	//if (!(fAccess & IEM_ACCESS_TYPE_READ))
4148	uErr \|= X86_TRAP_PF_RW;
4149	}
4150	#endif
4151
4152	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR \| IEM_XCPT_FLAGS_CR2,
4153	uErr, GCPtrWhere);
4154	}
4155
4156	#ifdef IEM_WITH_SETJMP
4157	/** \#PF(n) - 0e, longjmp. */
4158	DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT
4159	{
4160	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, fAccess, rc)));
4161	}
4162	#endif
4163
4164
4165	/** \#MF(0) - 10. */
4166	VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu)
4167	{
4168	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4169	}
4170
4171
4172	/** \#AC(0) - 11. */
4173	VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu)
4174	{
4175	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4176	}
4177
4178	#ifdef IEM_WITH_SETJMP
4179	/** \#AC(0) - 11, longjmp. */
4180	DECL_NO_RETURN(void) iemRaiseAlignmentCheckExceptionJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
4181	{
4182	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaiseAlignmentCheckException(pVCpu)));
4183	}
4184	#endif
4185
4186
4187	/** \#XF(0)/\#XM(0) - 19. */
4188	VBOXSTRICTRC iemRaiseSimdFpException(PVMCPUCC pVCpu) RT_NOEXCEPT
4189	{
4190	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_XF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4191	}
4192
4193
4194	/** Accessed via IEMOP_RAISE_DIVIDE_ERROR. */
4195	IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
4196	{
4197	NOREF(cbInstr);
4198	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4199	}
4200
4201
4202	/** Accessed via IEMOP_RAISE_INVALID_LOCK_PREFIX. */
4203	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
4204	{
4205	NOREF(cbInstr);
4206	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4207	}
4208
4209
4210	/** Accessed via IEMOP_RAISE_INVALID_OPCODE. */
4211	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
4212	{
4213	NOREF(cbInstr);
4214	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4215	}
4216
4217
4218	/** @} */
4219
4220	/** @name Common opcode decoders.
4221	* @{
4222	*/
4223	//#include <iprt/mem.h>
4224
4225	/**
4226	* Used to add extra details about a stub case.
4227	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4228	*/
4229	void iemOpStubMsg2(PVMCPUCC pVCpu) RT_NOEXCEPT
4230	{
4231	#if defined(LOG_ENABLED) && defined(IN_RING3)
4232	PVM pVM = pVCpu->CTX_SUFF(pVM);
4233	char szRegs[4096];
4234	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
4235	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
4236	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
4237	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
4238	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
4239	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
4240	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
4241	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
4242	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
4243	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
4244	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
4245	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
4246	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
4247	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
4248	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
4249	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
4250	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
4251	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
4252	" efer=%016VR{efer}\n"
4253	" pat=%016VR{pat}\n"
4254	" sf_mask=%016VR{sf_mask}\n"
4255	"krnl_gs_base=%016VR{krnl_gs_base}\n"
4256	" lstar=%016VR{lstar}\n"
4257	" star=%016VR{star} cstar=%016VR{cstar}\n"
4258	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
4259	);
4260
4261	char szInstr[256];
4262	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
4263	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
4264	szInstr, sizeof(szInstr), NULL);
4265
4266	RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
4267	#else
4268	RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip);
4269	#endif
4270	}
4271
4272	/** @} */
4273
4274
4275
4276	/** @name Register Access.
4277	* @{
4278	*/
4279
4280	/**
4281	* Adds a 8-bit signed jump offset to RIP/EIP/IP.
4282	*
4283	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4284	* segment limit.
4285	*
4286	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4287	* @param offNextInstr The offset of the next instruction.
4288	*/
4289	VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr) RT_NOEXCEPT
4290	{
4291	switch (pVCpu->iem.s.enmEffOpSize)
4292	{
4293	case IEMMODE_16BIT:
4294	{
4295	uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4296	if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
4297	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4298	return iemRaiseGeneralProtectionFault0(pVCpu);
4299	pVCpu->cpum.GstCtx.rip = uNewIp;
4300	break;
4301	}
4302
4303	case IEMMODE_32BIT:
4304	{
4305	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
4306	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4307
4308	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4309	if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
4310	return iemRaiseGeneralProtectionFault0(pVCpu);
4311	pVCpu->cpum.GstCtx.rip = uNewEip;
4312	break;
4313	}
4314
4315	case IEMMODE_64BIT:
4316	{
4317	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4318
4319	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4320	if (!IEM_IS_CANONICAL(uNewRip))
4321	return iemRaiseGeneralProtectionFault0(pVCpu);
4322	pVCpu->cpum.GstCtx.rip = uNewRip;
4323	break;
4324	}
4325
4326	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4327	}
4328
4329	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4330
4331	#ifndef IEM_WITH_CODE_TLB
4332	/* Flush the prefetch buffer. */
4333	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4334	#endif
4335
4336	return VINF_SUCCESS;
4337	}
4338
4339
4340	/**
4341	* Adds a 16-bit signed jump offset to RIP/EIP/IP.
4342	*
4343	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4344	* segment limit.
4345	*
4346	* @returns Strict VBox status code.
4347	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4348	* @param offNextInstr The offset of the next instruction.
4349	*/
4350	VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr) RT_NOEXCEPT
4351	{
4352	Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT);
4353
4354	uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4355	if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
4356	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4357	return iemRaiseGeneralProtectionFault0(pVCpu);
4358	/** @todo Test 16-bit jump in 64-bit mode. possible? */
4359	pVCpu->cpum.GstCtx.rip = uNewIp;
4360	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4361
4362	#ifndef IEM_WITH_CODE_TLB
4363	/* Flush the prefetch buffer. */
4364	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4365	#endif
4366
4367	return VINF_SUCCESS;
4368	}
4369
4370
4371	/**
4372	* Adds a 32-bit signed jump offset to RIP/EIP/IP.
4373	*
4374	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4375	* segment limit.
4376	*
4377	* @returns Strict VBox status code.
4378	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4379	* @param offNextInstr The offset of the next instruction.
4380	*/
4381	VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr) RT_NOEXCEPT
4382	{
4383	Assert(pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT);
4384
4385	if (pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT)
4386	{
4387	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4388
4389	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4390	if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
4391	return iemRaiseGeneralProtectionFault0(pVCpu);
4392	pVCpu->cpum.GstCtx.rip = uNewEip;
4393	}
4394	else
4395	{
4396	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4397
4398	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4399	if (!IEM_IS_CANONICAL(uNewRip))
4400	return iemRaiseGeneralProtectionFault0(pVCpu);
4401	pVCpu->cpum.GstCtx.rip = uNewRip;
4402	}
4403	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4404
4405	#ifndef IEM_WITH_CODE_TLB
4406	/* Flush the prefetch buffer. */
4407	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4408	#endif
4409
4410	return VINF_SUCCESS;
4411	}
4412
4413
4414	/**
4415	* Performs a near jump to the specified address.
4416	*
4417	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4418	* segment limit.
4419	*
4420	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4421	* @param uNewRip The new RIP value.
4422	*/
4423	VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT
4424	{
4425	switch (pVCpu->iem.s.enmEffOpSize)
4426	{
4427	case IEMMODE_16BIT:
4428	{
4429	Assert(uNewRip <= UINT16_MAX);
4430	if ( uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit
4431	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4432	return iemRaiseGeneralProtectionFault0(pVCpu);
4433	/** @todo Test 16-bit jump in 64-bit mode. */
4434	pVCpu->cpum.GstCtx.rip = uNewRip;
4435	break;
4436	}
4437
4438	case IEMMODE_32BIT:
4439	{
4440	Assert(uNewRip <= UINT32_MAX);
4441	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
4442	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4443
4444	if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
4445	return iemRaiseGeneralProtectionFault0(pVCpu);
4446	pVCpu->cpum.GstCtx.rip = uNewRip;
4447	break;
4448	}
4449
4450	case IEMMODE_64BIT:
4451	{
4452	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4453
4454	if (!IEM_IS_CANONICAL(uNewRip))
4455	return iemRaiseGeneralProtectionFault0(pVCpu);
4456	pVCpu->cpum.GstCtx.rip = uNewRip;
4457	break;
4458	}
4459
4460	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4461	}
4462
4463	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4464
4465	#ifndef IEM_WITH_CODE_TLB
4466	/* Flush the prefetch buffer. */
4467	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4468	#endif
4469
4470	return VINF_SUCCESS;
4471	}
4472
4473	/** @} */
4474
4475
4476	/** @name FPU access and helpers.
4477	*
4478	* @{
4479	*/
4480
4481	/**
4482	* Updates the x87.DS and FPUDP registers.
4483	*
4484	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4485	* @param pFpuCtx The FPU context.
4486	* @param iEffSeg The effective segment register.
4487	* @param GCPtrEff The effective address relative to @a iEffSeg.
4488	*/
4489	DECLINLINE(void) iemFpuUpdateDP(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
4490	{
4491	RTSEL sel;
4492	switch (iEffSeg)
4493	{
4494	case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break;
4495	case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break;
4496	case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break;
4497	case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break;
4498	case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break;
4499	case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break;
4500	default:
4501	AssertMsgFailed(("%d\n", iEffSeg));
4502	sel = pVCpu->cpum.GstCtx.ds.Sel;
4503	}
4504	/** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */
4505	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
4506	{
4507	pFpuCtx->DS = 0;
4508	pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4);
4509	}
4510	else if (!IEM_IS_LONG_MODE(pVCpu))
4511	{
4512	pFpuCtx->DS = sel;
4513	pFpuCtx->FPUDP = GCPtrEff;
4514	}
4515	else
4516	(uint64_t )&pFpuCtx->FPUDP = GCPtrEff;
4517	}
4518
4519
4520	/**
4521	* Rotates the stack registers in the push direction.
4522	*
4523	* @param pFpuCtx The FPU context.
4524	* @remarks This is a complete waste of time, but fxsave stores the registers in
4525	* stack order.
4526	*/
4527	DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx)
4528	{
4529	RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80;
4530	pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80;
4531	pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80;
4532	pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80;
4533	pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80;
4534	pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80;
4535	pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80;
4536	pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80;
4537	pFpuCtx->aRegs[0].r80 = r80Tmp;
4538	}
4539
4540
4541	/**
4542	* Rotates the stack registers in the pop direction.
4543	*
4544	* @param pFpuCtx The FPU context.
4545	* @remarks This is a complete waste of time, but fxsave stores the registers in
4546	* stack order.
4547	*/
4548	DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx)
4549	{
4550	RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80;
4551	pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80;
4552	pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80;
4553	pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80;
4554	pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80;
4555	pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80;
4556	pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80;
4557	pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80;
4558	pFpuCtx->aRegs[7].r80 = r80Tmp;
4559	}
4560
4561
4562	/**
4563	* Updates FSW and pushes a FPU result onto the FPU stack if no pending
4564	* exception prevents it.
4565	*
4566	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4567	* @param pResult The FPU operation result to push.
4568	* @param pFpuCtx The FPU context.
4569	*/
4570	static void iemFpuMaybePushResult(PVMCPU pVCpu, PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
4571	{
4572	/* Update FSW and bail if there are pending exceptions afterwards. */
4573	uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
4574	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4575	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4576	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4577	{
4578	if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FCW & X86_FSW_ES))
4579	Log11(("iemFpuMaybePushResult: %04x:%08RX64: FSW %#x -> %#x\n",
4580	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
4581	pFpuCtx->FSW = fFsw;
4582	return;
4583	}
4584
4585	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
4586	if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
4587	{
4588	/* All is fine, push the actual value. */
4589	pFpuCtx->FTW \|= RT_BIT(iNewTop);
4590	pFpuCtx->aRegs[7].r80 = pResult->r80Result;
4591	}
4592	else if (pFpuCtx->FCW & X86_FCW_IM)
4593	{
4594	/* Masked stack overflow, push QNaN. */
4595	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
4596	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
4597	}
4598	else
4599	{
4600	/* Raise stack overflow, don't push anything. */
4601	pFpuCtx->FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
4602	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
4603	Log11(("iemFpuMaybePushResult: %04x:%08RX64: stack overflow (FSW=%#x)\n",
4604	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4605	return;
4606	}
4607
4608	fFsw &= ~X86_FSW_TOP_MASK;
4609	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
4610	pFpuCtx->FSW = fFsw;
4611
4612	iemFpuRotateStackPush(pFpuCtx);
4613	RT_NOREF(pVCpu);
4614	}
4615
4616
4617	/**
4618	* Stores a result in a FPU register and updates the FSW and FTW.
4619	*
4620	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4621	* @param pFpuCtx The FPU context.
4622	* @param pResult The result to store.
4623	* @param iStReg Which FPU register to store it in.
4624	*/
4625	static void iemFpuStoreResultOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4626	{
4627	Assert(iStReg < 8);
4628	uint16_t fNewFsw = pFpuCtx->FSW;
4629	uint16_t const iReg = (X86_FSW_TOP_GET(fNewFsw) + iStReg) & X86_FSW_TOP_SMASK;
4630	fNewFsw &= ~X86_FSW_C_MASK;
4631	fNewFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4632	if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4633	Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
4634	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
4635	pFpuCtx->FSW = fNewFsw;
4636	pFpuCtx->FTW \|= RT_BIT(iReg);
4637	pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result;
4638	RT_NOREF(pVCpu);
4639	}
4640
4641
4642	/**
4643	* Only updates the FPU status word (FSW) with the result of the current
4644	* instruction.
4645	*
4646	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4647	* @param pFpuCtx The FPU context.
4648	* @param u16FSW The FSW output of the current instruction.
4649	*/
4650	static void iemFpuUpdateFSWOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint16_t u16FSW) RT_NOEXCEPT
4651	{
4652	uint16_t fNewFsw = pFpuCtx->FSW;
4653	fNewFsw &= ~X86_FSW_C_MASK;
4654	fNewFsw \|= u16FSW & ~X86_FSW_TOP_MASK;
4655	if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4656	Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
4657	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
4658	pFpuCtx->FSW = fNewFsw;
4659	RT_NOREF(pVCpu);
4660	}
4661
4662
4663	/**
4664	* Pops one item off the FPU stack if no pending exception prevents it.
4665	*
4666	* @param pFpuCtx The FPU context.
4667	*/
4668	static void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
4669	{
4670	/* Check pending exceptions. */
4671	uint16_t uFSW = pFpuCtx->FSW;
4672	if ( (pFpuCtx->FSW & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4673	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4674	return;
4675
4676	/* TOP--. */
4677	uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
4678	uFSW &= ~X86_FSW_TOP_MASK;
4679	uFSW \|= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
4680	pFpuCtx->FSW = uFSW;
4681
4682	/* Mark the previous ST0 as empty. */
4683	iOldTop >>= X86_FSW_TOP_SHIFT;
4684	pFpuCtx->FTW &= ~RT_BIT(iOldTop);
4685
4686	/* Rotate the registers. */
4687	iemFpuRotateStackPop(pFpuCtx);
4688	}
4689
4690
4691	/**
4692	* Pushes a FPU result onto the FPU stack if no pending exception prevents it.
4693	*
4694	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4695	* @param pResult The FPU operation result to push.
4696	*/
4697	void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult) RT_NOEXCEPT
4698	{
4699	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4700	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4701	iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
4702	}
4703
4704
4705	/**
4706	* Pushes a FPU result onto the FPU stack if no pending exception prevents it,
4707	* and sets FPUDP and FPUDS.
4708	*
4709	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4710	* @param pResult The FPU operation result to push.
4711	* @param iEffSeg The effective segment register.
4712	* @param GCPtrEff The effective address relative to @a iEffSeg.
4713	*/
4714	void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4715	{
4716	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4717	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4718	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4719	iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
4720	}
4721
4722
4723	/**
4724	* Replace ST0 with the first value and push the second onto the FPU stack,
4725	* unless a pending exception prevents it.
4726	*
4727	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4728	* @param pResult The FPU operation result to store and push.
4729	*/
4730	void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult) RT_NOEXCEPT
4731	{
4732	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4733	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4734
4735	/* Update FSW and bail if there are pending exceptions afterwards. */
4736	uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
4737	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4738	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4739	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4740	{
4741	if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4742	Log11(("iemFpuPushResultTwo: %04x:%08RX64: FSW %#x -> %#x\n",
4743	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
4744	pFpuCtx->FSW = fFsw;
4745	return;
4746	}
4747
4748	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
4749	if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
4750	{
4751	/* All is fine, push the actual value. */
4752	pFpuCtx->FTW \|= RT_BIT(iNewTop);
4753	pFpuCtx->aRegs[0].r80 = pResult->r80Result1;
4754	pFpuCtx->aRegs[7].r80 = pResult->r80Result2;
4755	}
4756	else if (pFpuCtx->FCW & X86_FCW_IM)
4757	{
4758	/* Masked stack overflow, push QNaN. */
4759	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
4760	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
4761	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
4762	}
4763	else
4764	{
4765	/* Raise stack overflow, don't push anything. */
4766	pFpuCtx->FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
4767	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
4768	Log11(("iemFpuPushResultTwo: %04x:%08RX64: stack overflow (FSW=%#x)\n",
4769	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4770	return;
4771	}
4772
4773	fFsw &= ~X86_FSW_TOP_MASK;
4774	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
4775	pFpuCtx->FSW = fFsw;
4776
4777	iemFpuRotateStackPush(pFpuCtx);
4778	}
4779
4780
4781	/**
4782	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
4783	* FOP.
4784	*
4785	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4786	* @param pResult The result to store.
4787	* @param iStReg Which FPU register to store it in.
4788	*/
4789	void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4790	{
4791	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4792	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4793	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4794	}
4795
4796
4797	/**
4798	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
4799	* FOP, and then pops the stack.
4800	*
4801	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4802	* @param pResult The result to store.
4803	* @param iStReg Which FPU register to store it in.
4804	*/
4805	void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4806	{
4807	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4808	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4809	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4810	iemFpuMaybePopOne(pFpuCtx);
4811	}
4812
4813
4814	/**
4815	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
4816	* FPUDP, and FPUDS.
4817	*
4818	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4819	* @param pResult The result to store.
4820	* @param iStReg Which FPU register to store it in.
4821	* @param iEffSeg The effective memory operand selector register.
4822	* @param GCPtrEff The effective memory operand offset.
4823	*/
4824	void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
4825	uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4826	{
4827	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4828	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4829	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4830	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4831	}
4832
4833
4834	/**
4835	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
4836	* FPUDP, and FPUDS, and then pops the stack.
4837	*
4838	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4839	* @param pResult The result to store.
4840	* @param iStReg Which FPU register to store it in.
4841	* @param iEffSeg The effective memory operand selector register.
4842	* @param GCPtrEff The effective memory operand offset.
4843	*/
4844	void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult,
4845	uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4846	{
4847	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4848	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4849	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4850	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4851	iemFpuMaybePopOne(pFpuCtx);
4852	}
4853
4854
4855	/**
4856	* Updates the FOP, FPUIP, and FPUCS. For FNOP.
4857	*
4858	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4859	*/
4860	void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu) RT_NOEXCEPT
4861	{
4862	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4863	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4864	}
4865
4866
4867	/**
4868	* Updates the FSW, FOP, FPUIP, and FPUCS.
4869	*
4870	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4871	* @param u16FSW The FSW from the current instruction.
4872	*/
4873	void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4874	{
4875	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4876	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4877	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4878	}
4879
4880
4881	/**
4882	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
4883	*
4884	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4885	* @param u16FSW The FSW from the current instruction.
4886	*/
4887	void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4888	{
4889	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4890	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4891	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4892	iemFpuMaybePopOne(pFpuCtx);
4893	}
4894
4895
4896	/**
4897	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
4898	*
4899	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4900	* @param u16FSW The FSW from the current instruction.
4901	* @param iEffSeg The effective memory operand selector register.
4902	* @param GCPtrEff The effective memory operand offset.
4903	*/
4904	void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4905	{
4906	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4907	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4908	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4909	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4910	}
4911
4912
4913	/**
4914	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
4915	*
4916	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4917	* @param u16FSW The FSW from the current instruction.
4918	*/
4919	void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4920	{
4921	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4922	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4923	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4924	iemFpuMaybePopOne(pFpuCtx);
4925	iemFpuMaybePopOne(pFpuCtx);
4926	}
4927
4928
4929	/**
4930	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
4931	*
4932	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4933	* @param u16FSW The FSW from the current instruction.
4934	* @param iEffSeg The effective memory operand selector register.
4935	* @param GCPtrEff The effective memory operand offset.
4936	*/
4937	void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4938	{
4939	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4940	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4941	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4942	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4943	iemFpuMaybePopOne(pFpuCtx);
4944	}
4945
4946
4947	/**
4948	* Worker routine for raising an FPU stack underflow exception.
4949	*
4950	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4951	* @param pFpuCtx The FPU context.
4952	* @param iStReg The stack register being accessed.
4953	*/
4954	static void iemFpuStackUnderflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint8_t iStReg)
4955	{
4956	Assert(iStReg < 8 \|\| iStReg == UINT8_MAX);
4957	if (pFpuCtx->FCW & X86_FCW_IM)
4958	{
4959	/* Masked underflow. */
4960	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
4961	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
4962	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
4963	if (iStReg != UINT8_MAX)
4964	{
4965	pFpuCtx->FTW \|= RT_BIT(iReg);
4966	iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
4967	}
4968	}
4969	else
4970	{
4971	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
4972	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
4973	Log11(("iemFpuStackUnderflowOnly: %04x:%08RX64: underflow (FSW=%#x)\n",
4974	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4975	}
4976	RT_NOREF(pVCpu);
4977	}
4978
4979
4980	/**
4981	* Raises a FPU stack underflow exception.
4982	*
4983	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4984	* @param iStReg The destination register that should be loaded
4985	* with QNaN if \#IS is not masked. Specify
4986	* UINT8_MAX if none (like for fcom).
4987	*/
4988	void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
4989	{
4990	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4991	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4992	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
4993	}
4994
4995
4996	void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4997	{
4998	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4999	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5000	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5001	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5002	}
5003
5004
5005	void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
5006	{
5007	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5008	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5009	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5010	iemFpuMaybePopOne(pFpuCtx);
5011	}
5012
5013
5014	void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
5015	{
5016	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5017	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5018	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5019	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5020	iemFpuMaybePopOne(pFpuCtx);
5021	}
5022
5023
5024	void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu) RT_NOEXCEPT
5025	{
5026	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5027	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5028	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, UINT8_MAX);
5029	iemFpuMaybePopOne(pFpuCtx);
5030	iemFpuMaybePopOne(pFpuCtx);
5031	}
5032
5033
5034	void iemFpuStackPushUnderflow(PVMCPUCC pVCpu) RT_NOEXCEPT
5035	{
5036	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5037	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5038
5039	if (pFpuCtx->FCW & X86_FCW_IM)
5040	{
5041	/* Masked overflow - Push QNaN. */
5042	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5043	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5044	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
5045	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5046	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5047	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5048	iemFpuRotateStackPush(pFpuCtx);
5049	}
5050	else
5051	{
5052	/* Exception pending - don't change TOP or the register stack. */
5053	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5054	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5055	Log11(("iemFpuStackPushUnderflow: %04x:%08RX64: underflow (FSW=%#x)\n",
5056	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5057	}
5058	}
5059
5060
5061	void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu) RT_NOEXCEPT
5062	{
5063	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5064	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5065
5066	if (pFpuCtx->FCW & X86_FCW_IM)
5067	{
5068	/* Masked overflow - Push QNaN. */
5069	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5070	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5071	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
5072	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5073	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5074	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
5075	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5076	iemFpuRotateStackPush(pFpuCtx);
5077	}
5078	else
5079	{
5080	/* Exception pending - don't change TOP or the register stack. */
5081	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5082	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5083	Log11(("iemFpuStackPushUnderflowTwo: %04x:%08RX64: underflow (FSW=%#x)\n",
5084	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5085	}
5086	}
5087
5088
5089	/**
5090	* Worker routine for raising an FPU stack overflow exception on a push.
5091	*
5092	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5093	* @param pFpuCtx The FPU context.
5094	*/
5095	static void iemFpuStackPushOverflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
5096	{
5097	if (pFpuCtx->FCW & X86_FCW_IM)
5098	{
5099	/* Masked overflow. */
5100	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5101	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5102	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
5103	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5104	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5105	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5106	iemFpuRotateStackPush(pFpuCtx);
5107	}
5108	else
5109	{
5110	/* Exception pending - don't change TOP or the register stack. */
5111	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5112	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5113	Log11(("iemFpuStackPushOverflowOnly: %04x:%08RX64: overflow (FSW=%#x)\n",
5114	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5115	}
5116	RT_NOREF(pVCpu);
5117	}
5118
5119
5120	/**
5121	* Raises a FPU stack overflow exception on a push.
5122	*
5123	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5124	*/
5125	void iemFpuStackPushOverflow(PVMCPUCC pVCpu) RT_NOEXCEPT
5126	{
5127	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5128	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5129	iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
5130	}
5131
5132
5133	/**
5134	* Raises a FPU stack overflow exception on a push with a memory operand.
5135	*
5136	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5137	* @param iEffSeg The effective memory operand selector register.
5138	* @param GCPtrEff The effective memory operand offset.
5139	*/
5140	void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
5141	{
5142	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5143	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5144	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5145	iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
5146	}
5147
5148	/** @} */
5149
5150
5151	/** @name SSE+AVX SIMD access and helpers.
5152	*
5153	* @{
5154	*/
5155	/**
5156	* Stores a result in a SIMD XMM register, updates the MXCSR.
5157	*
5158	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5159	* @param pResult The result to store.
5160	* @param iXmmReg Which SIMD XMM register to store the result in.
5161	*/
5162	void iemSseStoreResult(PVMCPUCC pVCpu, PCIEMSSERESULT pResult, uint8_t iXmmReg) RT_NOEXCEPT
5163	{
5164	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5165	pFpuCtx->MXCSR \|= pResult->MXCSR & X86_MXCSR_XCPT_FLAGS;
5166	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXmmReg] = pResult->uResult;
5167	}
5168
5169	/** @} */
5170
5171
5172	/** @name Memory access.
5173	*
5174	* @{
5175	*/
5176
5177
5178	/**
5179	* Updates the IEMCPU::cbWritten counter if applicable.
5180	*
5181	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5182	* @param fAccess The access being accounted for.
5183	* @param cbMem The access size.
5184	*/
5185	DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPUCC pVCpu, uint32_t fAccess, size_t cbMem)
5186	{
5187	if ( (fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK \| IEM_ACCESS_TYPE_WRITE)
5188	\|\| (fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA \| IEM_ACCESS_TYPE_WRITE) )
5189	pVCpu->iem.s.cbWritten += (uint32_t)cbMem;
5190	}
5191
5192
5193	/**
5194	* Applies the segment limit, base and attributes.
5195	*
5196	* This may raise a \#GP or \#SS.
5197	*
5198	* @returns VBox strict status code.
5199	*
5200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5201	* @param fAccess The kind of access which is being performed.
5202	* @param iSegReg The index of the segment register to apply.
5203	* This is UINT8_MAX if none (for IDT, GDT, LDT,
5204	* TSS, ++).
5205	* @param cbMem The access size.
5206	* @param pGCPtrMem Pointer to the guest memory address to apply
5207	* segmentation to. Input and output parameter.
5208	*/
5209	VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT
5210	{
5211	if (iSegReg == UINT8_MAX)
5212	return VINF_SUCCESS;
5213
5214	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
5215	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
5216	switch (pVCpu->iem.s.enmCpuMode)
5217	{
5218	case IEMMODE_16BIT:
5219	case IEMMODE_32BIT:
5220	{
5221	RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
5222	RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
5223
5224	if ( pSel->Attr.n.u1Present
5225	&& !pSel->Attr.n.u1Unusable)
5226	{
5227	Assert(pSel->Attr.n.u1DescType);
5228	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5229	{
5230	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5231	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
5232	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
5233
5234	if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
5235	{
5236	/** @todo CPL check. */
5237	}
5238
5239	/*
5240	* There are two kinds of data selectors, normal and expand down.
5241	*/
5242	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5243	{
5244	if ( GCPtrFirst32 > pSel->u32Limit
5245	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
5246	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5247	}
5248	else
5249	{
5250	/*
5251	* The upper boundary is defined by the B bit, not the G bit!
5252	*/
5253	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5254	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5255	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5256	}
5257	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
5258	}
5259	else
5260	{
5261	/*
5262	* Code selector and usually be used to read thru, writing is
5263	* only permitted in real and V8086 mode.
5264	*/
5265	if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5266	\|\| ( (fAccess & IEM_ACCESS_TYPE_READ)
5267	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
5268	&& !IEM_IS_REAL_OR_V86_MODE(pVCpu) )
5269	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
5270
5271	if ( GCPtrFirst32 > pSel->u32Limit
5272	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
5273	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5274
5275	if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
5276	{
5277	/** @todo CPL check. */
5278	}
5279
5280	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
5281	}
5282	}
5283	else
5284	return iemRaiseGeneralProtectionFault0(pVCpu);
5285	return VINF_SUCCESS;
5286	}
5287
5288	case IEMMODE_64BIT:
5289	{
5290	RTGCPTR GCPtrMem = *pGCPtrMem;
5291	if (iSegReg == X86_SREG_GS \|\| iSegReg == X86_SREG_FS)
5292	*pGCPtrMem = GCPtrMem + pSel->u64Base;
5293
5294	Assert(cbMem >= 1);
5295	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
5296	return VINF_SUCCESS;
5297	/** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec.
5298	* 4.12.2 "Data Limit Checks in 64-bit Mode". */
5299	return iemRaiseGeneralProtectionFault0(pVCpu);
5300	}
5301
5302	default:
5303	AssertFailedReturn(VERR_IEM_IPE_7);
5304	}
5305	}
5306
5307
5308	/**
5309	* Translates a virtual address to a physical physical address and checks if we
5310	* can access the page as specified.
5311	*
5312	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5313	* @param GCPtrMem The virtual address.
5314	* @param fAccess The intended access.
5315	* @param pGCPhysMem Where to return the physical address.
5316	*/
5317	VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT
5318	{
5319	/** @todo Need a different PGM interface here. We're currently using
5320	* generic / REM interfaces. this won't cut it for R0. */
5321	/** @todo If/when PGM handles paged real-mode, we can remove the hack in
5322	* iemSvmWorldSwitch/iemVmxWorldSwitch to work around raising a page-fault
5323	* here. */
5324	PGMPTWALK Walk;
5325	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
5326	if (RT_FAILURE(rc))
5327	{
5328	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
5329	/** @todo Check unassigned memory in unpaged mode. */
5330	/** @todo Reserved bits in page tables. Requires new PGM interface. */
5331	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5332	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5333	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
5334	#endif
5335	*pGCPhysMem = NIL_RTGCPHYS;
5336	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, rc);
5337	}
5338
5339	/* If the page is writable and does not have the no-exec bit set, all
5340	access is allowed. Otherwise we'll have to check more carefully... */
5341	if ((Walk.fEffective & (X86_PTE_RW \| X86_PTE_US \| X86_PTE_PAE_NX)) != (X86_PTE_RW \| X86_PTE_US))
5342	{
5343	/* Write to read only memory? */
5344	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5345	&& !(Walk.fEffective & X86_PTE_RW)
5346	&& ( ( pVCpu->iem.s.uCpl == 3
5347	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
5348	\|\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
5349	{
5350	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
5351	*pGCPhysMem = NIL_RTGCPHYS;
5352	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5353	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5354	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5355	#endif
5356	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
5357	}
5358
5359	/* Kernel memory accessed by userland? */
5360	if ( !(Walk.fEffective & X86_PTE_US)
5361	&& pVCpu->iem.s.uCpl == 3
5362	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
5363	{
5364	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
5365	*pGCPhysMem = NIL_RTGCPHYS;
5366	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5367	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5368	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5369	#endif
5370	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
5371	}
5372
5373	/* Executing non-executable memory? */
5374	if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
5375	&& (Walk.fEffective & X86_PTE_PAE_NX)
5376	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) )
5377	{
5378	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
5379	*pGCPhysMem = NIL_RTGCPHYS;
5380	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5381	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5382	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5383	#endif
5384	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE),
5385	VERR_ACCESS_DENIED);
5386	}
5387	}
5388
5389	/*
5390	* Set the dirty / access flags.
5391	* ASSUMES this is set when the address is translated rather than on committ...
5392	*/
5393	/** @todo testcase: check when A and D bits are actually set by the CPU. */
5394	uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
5395	if ((Walk.fEffective & fAccessedDirty) != fAccessedDirty)
5396	{
5397	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
5398	AssertRC(rc2);
5399	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
5400	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
5401	}
5402
5403	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
5404	*pGCPhysMem = GCPhys;
5405	return VINF_SUCCESS;
5406	}
5407
5408
5409	/**
5410	* Looks up a memory mapping entry.
5411	*
5412	* @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
5413	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5414	* @param pvMem The memory address.
5415	* @param fAccess The access to.
5416	*/
5417	DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
5418	{
5419	Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
5420	fAccess &= IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK;
5421	if ( pVCpu->iem.s.aMemMappings[0].pv == pvMem
5422	&& (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5423	return 0;
5424	if ( pVCpu->iem.s.aMemMappings[1].pv == pvMem
5425	&& (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5426	return 1;
5427	if ( pVCpu->iem.s.aMemMappings[2].pv == pvMem
5428	&& (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5429	return 2;
5430	return VERR_NOT_FOUND;
5431	}
5432
5433
5434	/**
5435	* Finds a free memmap entry when using iNextMapping doesn't work.
5436	*
5437	* @returns Memory mapping index, 1024 on failure.
5438	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5439	*/
5440	static unsigned iemMemMapFindFree(PVMCPUCC pVCpu)
5441	{
5442	/*
5443	* The easy case.
5444	*/
5445	if (pVCpu->iem.s.cActiveMappings == 0)
5446	{
5447	pVCpu->iem.s.iNextMapping = 1;
5448	return 0;
5449	}
5450
5451	/* There should be enough mappings for all instructions. */
5452	AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);
5453
5454	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)
5455	if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
5456	return i;
5457
5458	AssertFailedReturn(1024);
5459	}
5460
5461
5462	/**
5463	* Commits a bounce buffer that needs writing back and unmaps it.
5464	*
5465	* @returns Strict VBox status code.
5466	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5467	* @param iMemMap The index of the buffer to commit.
5468	* @param fPostponeFail Whether we can postpone writer failures to ring-3.
5469	* Always false in ring-3, obviously.
5470	*/
5471	static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail)
5472	{
5473	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
5474	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
5475	#ifdef IN_RING3
5476	Assert(!fPostponeFail);
5477	RT_NOREF_PV(fPostponeFail);
5478	#endif
5479
5480	/*
5481	* Do the writing.
5482	*/
5483	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5484	if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)
5485	{
5486	uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
5487	uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
5488	uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5489	if (!pVCpu->iem.s.fBypassHandlers)
5490	{
5491	/*
5492	* Carefully and efficiently dealing with access handler return
5493	* codes make this a little bloated.
5494	*/
5495	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,
5496	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
5497	pbBuf,
5498	cbFirst,
5499	PGMACCESSORIGIN_IEM);
5500	if (rcStrict == VINF_SUCCESS)
5501	{
5502	if (cbSecond)
5503	{
5504	rcStrict = PGMPhysWrite(pVM,
5505	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5506	pbBuf + cbFirst,
5507	cbSecond,
5508	PGMACCESSORIGIN_IEM);
5509	if (rcStrict == VINF_SUCCESS)
5510	{ /* nothing */ }
5511	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5512	{
5513	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",
5514	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5515	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5516	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5517	}
5518	#ifndef IN_RING3
5519	else if (fPostponeFail)
5520	{
5521	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5522	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5523	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5524	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_2ND;
5525	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5526	return iemSetPassUpStatus(pVCpu, rcStrict);
5527	}
5528	#endif
5529	else
5530	{
5531	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5532	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5533	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5534	return rcStrict;
5535	}
5536	}
5537	}
5538	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5539	{
5540	if (!cbSecond)
5541	{
5542	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",
5543	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5544	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5545	}
5546	else
5547	{
5548	VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,
5549	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5550	pbBuf + cbFirst,
5551	cbSecond,
5552	PGMACCESSORIGIN_IEM);
5553	if (rcStrict2 == VINF_SUCCESS)
5554	{
5555	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",
5556	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5557	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5558	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5559	}
5560	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
5561	{
5562	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",
5563	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5564	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
5565	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
5566	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5567	}
5568	#ifndef IN_RING3
5569	else if (fPostponeFail)
5570	{
5571	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5572	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5573	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5574	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_2ND;
5575	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5576	return iemSetPassUpStatus(pVCpu, rcStrict);
5577	}
5578	#endif
5579	else
5580	{
5581	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5582	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5583	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
5584	return rcStrict2;
5585	}
5586	}
5587	}
5588	#ifndef IN_RING3
5589	else if (fPostponeFail)
5590	{
5591	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5592	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5593	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5594	if (!cbSecond)
5595	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_1ST;
5596	else
5597	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND;
5598	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5599	return iemSetPassUpStatus(pVCpu, rcStrict);
5600	}
5601	#endif
5602	else
5603	{
5604	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
5605	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5606	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5607	return rcStrict;
5608	}
5609	}
5610	else
5611	{
5612	/*
5613	* No access handlers, much simpler.
5614	*/
5615	int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);
5616	if (RT_SUCCESS(rc))
5617	{
5618	if (cbSecond)
5619	{
5620	rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);
5621	if (RT_SUCCESS(rc))
5622	{ /* likely */ }
5623	else
5624	{
5625	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5626	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5627	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
5628	return rc;
5629	}
5630	}
5631	}
5632	else
5633	{
5634	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
5635	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,
5636	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5637	return rc;
5638	}
5639	}
5640	}
5641
5642	#if defined(IEM_LOG_MEMORY_WRITES)
5643	Log(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
5644	RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));
5645	if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)
5646	Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5647	RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),
5648	&pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));
5649
5650	size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
5651	g_cbIemWrote = cbWrote;
5652	memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
5653	#endif
5654
5655	/*
5656	* Free the mapping entry.
5657	*/
5658	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
5659	Assert(pVCpu->iem.s.cActiveMappings != 0);
5660	pVCpu->iem.s.cActiveMappings--;
5661	return VINF_SUCCESS;
5662	}
5663
5664
5665	/**
5666	* iemMemMap worker that deals with a request crossing pages.
5667	*/
5668	static VBOXSTRICTRC
5669	iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
5670	{
5671	/*
5672	* Do the address translations.
5673	*/
5674	RTGCPHYS GCPhysFirst;
5675	VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, fAccess, &GCPhysFirst);
5676	if (rcStrict != VINF_SUCCESS)
5677	return rcStrict;
5678
5679	RTGCPHYS GCPhysSecond;
5680	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK,
5681	fAccess, &GCPhysSecond);
5682	if (rcStrict != VINF_SUCCESS)
5683	return rcStrict;
5684	GCPhysSecond &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
5685
5686	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5687
5688	/*
5689	* Read in the current memory content if it's a read, execute or partial
5690	* write access.
5691	*/
5692	uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5693	uint32_t const cbFirstPage = GUEST_PAGE_SIZE - (GCPhysFirst & GUEST_PAGE_OFFSET_MASK);
5694	uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
5695
5696	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
5697	{
5698	if (!pVCpu->iem.s.fBypassHandlers)
5699	{
5700	/*
5701	* Must carefully deal with access handler status codes here,
5702	* makes the code a bit bloated.
5703	*/
5704	rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);
5705	if (rcStrict == VINF_SUCCESS)
5706	{
5707	rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
5708	if (rcStrict == VINF_SUCCESS)
5709	{ /likely / }
5710	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5711	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5712	else
5713	{
5714	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",
5715	GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5716	return rcStrict;
5717	}
5718	}
5719	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5720	{
5721	VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
5722	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
5723	{
5724	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
5725	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5726	}
5727	else
5728	{
5729	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",
5730	GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));
5731	return rcStrict2;
5732	}
5733	}
5734	else
5735	{
5736	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5737	GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5738	return rcStrict;
5739	}
5740	}
5741	else
5742	{
5743	/*
5744	* No informational status codes here, much more straight forward.
5745	*/
5746	int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);
5747	if (RT_SUCCESS(rc))
5748	{
5749	Assert(rc == VINF_SUCCESS);
5750	rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
5751	if (RT_SUCCESS(rc))
5752	Assert(rc == VINF_SUCCESS);
5753	else
5754	{
5755	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
5756	return rc;
5757	}
5758	}
5759	else
5760	{
5761	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
5762	return rc;
5763	}
5764	}
5765	}
5766	#ifdef VBOX_STRICT
5767	else
5768	memset(pbBuf, 0xcc, cbMem);
5769	if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
5770	memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
5771	#endif
5772
5773	/*
5774	* Commit the bounce buffer entry.
5775	*/
5776	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
5777	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
5778	pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
5779	pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
5780	pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = false;
5781	pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
5782	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
5783	pVCpu->iem.s.iNextMapping = iMemMap + 1;
5784	pVCpu->iem.s.cActiveMappings++;
5785
5786	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
5787	*ppvMem = pbBuf;
5788	return VINF_SUCCESS;
5789	}
5790
5791
5792	/**
5793	* iemMemMap woker that deals with iemMemPageMap failures.
5794	*/
5795	static VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
5796	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
5797	{
5798	/*
5799	* Filter out conditions we can handle and the ones which shouldn't happen.
5800	*/
5801	if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
5802	&& rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
5803	&& rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
5804	{
5805	AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);
5806	return rcMap;
5807	}
5808	pVCpu->iem.s.cPotentialExits++;
5809
5810	/*
5811	* Read in the current memory content if it's a read, execute or partial
5812	* write access.
5813	*/
5814	uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5815	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
5816	{
5817	if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
5818	memset(pbBuf, 0xff, cbMem);
5819	else
5820	{
5821	int rc;
5822	if (!pVCpu->iem.s.fBypassHandlers)
5823	{
5824	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);
5825	if (rcStrict == VINF_SUCCESS)
5826	{ /* nothing */ }
5827	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5828	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5829	else
5830	{
5831	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5832	GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5833	return rcStrict;
5834	}
5835	}
5836	else
5837	{
5838	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);
5839	if (RT_SUCCESS(rc))
5840	{ /* likely */ }
5841	else
5842	{
5843	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5844	GCPhysFirst, rc));
5845	return rc;
5846	}
5847	}
5848	}
5849	}
5850	#ifdef VBOX_STRICT
5851	else
5852	memset(pbBuf, 0xcc, cbMem);
5853	#endif
5854	#ifdef VBOX_STRICT
5855	if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
5856	memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
5857	#endif
5858
5859	/*
5860	* Commit the bounce buffer entry.
5861	*/
5862	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
5863	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
5864	pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
5865	pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = 0;
5866	pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
5867	pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
5868	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
5869	pVCpu->iem.s.iNextMapping = iMemMap + 1;
5870	pVCpu->iem.s.cActiveMappings++;
5871
5872	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
5873	*ppvMem = pbBuf;
5874	return VINF_SUCCESS;
5875	}
5876
5877
5878
5879	/**
5880	* Maps the specified guest memory for the given kind of access.
5881	*
5882	* This may be using bounce buffering of the memory if it's crossing a page
5883	* boundary or if there is an access handler installed for any of it. Because
5884	* of lock prefix guarantees, we're in for some extra clutter when this
5885	* happens.
5886	*
5887	* This may raise a \#GP, \#SS, \#PF or \#AC.
5888	*
5889	* @returns VBox strict status code.
5890	*
5891	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5892	* @param ppvMem Where to return the pointer to the mapped memory.
5893	* @param cbMem The number of bytes to map. This is usually 1, 2, 4, 6,
5894	* 8, 12, 16, 32 or 512. When used by string operations
5895	* it can be up to a page.
5896	* @param iSegReg The index of the segment register to use for this
5897	* access. The base and limits are checked. Use UINT8_MAX
5898	* to indicate that no segmentation is required (for IDT,
5899	* GDT and LDT accesses).
5900	* @param GCPtrMem The address of the guest memory.
5901	* @param fAccess How the memory is being accessed. The
5902	* IEM_ACCESS_TYPE_XXX bit is used to figure out how to map
5903	* the memory, while the IEM_ACCESS_WHAT_XXX bit is used
5904	* when raising exceptions.
5905	* @param uAlignCtl Alignment control:
5906	* - Bits 15:0 is the alignment mask.
5907	* - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
5908	* IEM_MEMMAP_F_ALIGN_SSE, and
5909	* IEM_MEMMAP_F_ALIGN_GP_OR_AC.
5910	* Pass zero to skip alignment.
5911	*/
5912	VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem,
5913	uint32_t fAccess, uint32_t uAlignCtl) RT_NOEXCEPT
5914	{
5915	/*
5916	* Check the input and figure out which mapping entry to use.
5917	*/
5918	Assert(cbMem <= 64 \|\| cbMem == 512 \|\| cbMem == 256 \|\| cbMem == 108 \|\| cbMem == 104 \|\| cbMem == 102 \|\| cbMem == 94); /* 512 is the max! */
5919	Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK)));
5920	Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
5921
5922	unsigned iMemMap = pVCpu->iem.s.iNextMapping;
5923	if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
5924	\|\| pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
5925	{
5926	iMemMap = iemMemMapFindFree(pVCpu);
5927	AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
5928	("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
5929	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
5930	pVCpu->iem.s.aMemMappings[2].fAccess),
5931	VERR_IEM_IPE_9);
5932	}
5933
5934	/*
5935	* Map the memory, checking that we can actually access it. If something
5936	* slightly complicated happens, fall back on bounce buffering.
5937	*/
5938	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
5939	if (rcStrict == VINF_SUCCESS)
5940	{ /* likely */ }
5941	else
5942	return rcStrict;
5943
5944	if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE) /* Crossing a page boundary? */
5945	{ /* likely */ }
5946	else
5947	return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
5948
5949	/*
5950	* Alignment check.
5951	*/
5952	if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
5953	{ /* likelyish */ }
5954	else
5955	{
5956	/* Misaligned access. */
5957	if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
5958	{
5959	if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
5960	\|\| ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
5961	&& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
5962	{
5963	AssertCompile(X86_CR0_AM == X86_EFL_AC);
5964
5965	if (iemMemAreAlignmentChecksEnabled(pVCpu))
5966	return iemRaiseAlignmentCheckException(pVCpu);
5967	}
5968	else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
5969	&& iemMemAreAlignmentChecksEnabled(pVCpu)
5970	/** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
5971	* implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. */
5972	)
5973	return iemRaiseAlignmentCheckException(pVCpu);
5974	else
5975	return iemRaiseGeneralProtectionFault0(pVCpu);
5976	}
5977	}
5978
5979	#ifdef IEM_WITH_DATA_TLB
5980	Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
5981
5982	/*
5983	* Get the TLB entry for this page.
5984	*/
5985	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem);
5986	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
5987	if (pTlbe->uTag == uTag)
5988	{
5989	# ifdef VBOX_WITH_STATISTICS
5990	pVCpu->iem.s.DataTlb.cTlbHits++;
5991	# endif
5992	}
5993	else
5994	{
5995	pVCpu->iem.s.DataTlb.cTlbMisses++;
5996	PGMPTWALK Walk;
5997	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
5998	if (RT_FAILURE(rc))
5999	{
6000	Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
6001	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6002	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6003	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
6004	# endif
6005	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, rc);
6006	}
6007
6008	Assert(Walk.fSucceeded);
6009	pTlbe->uTag = uTag;
6010	pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A); /* skipping NX */
6011	pTlbe->GCPhys = Walk.GCPhys;
6012	pTlbe->pbMappingR3 = NULL;
6013	}
6014
6015	/*
6016	* Check TLB page table level access flags.
6017	*/
6018	/* If the page is either supervisor only or non-writable, we need to do
6019	more careful access checks. */
6020	if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER \| IEMTLBE_F_PT_NO_WRITE))
6021	{
6022	/* Write to read only memory? */
6023	if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_WRITE)
6024	&& (fAccess & IEM_ACCESS_TYPE_WRITE)
6025	&& ( ( pVCpu->iem.s.uCpl == 3
6026	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
6027	\|\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
6028	{
6029	Log(("iemMemMap: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6030	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6031	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6032	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6033	# endif
6034	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6035	}
6036
6037	/* Kernel memory accessed by userland? */
6038	if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER)
6039	&& pVCpu->iem.s.uCpl == 3
6040	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
6041	{
6042	Log(("iemMemMap: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6043	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6044	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6045	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6046	# endif
6047	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
6048	}
6049	}
6050
6051	/*
6052	* Set the dirty / access flags.
6053	* ASSUMES this is set when the address is translated rather than on commit...
6054	*/
6055	/** @todo testcase: check when A and D bits are actually set by the CPU. */
6056	uint64_t const fTlbAccessedDirty = (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PT_NO_DIRTY : 0) \| IEMTLBE_F_PT_NO_ACCESSED;
6057	if (pTlbe->fFlagsAndPhysRev & fTlbAccessedDirty)
6058	{
6059	uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
6060	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6061	AssertRC(rc2);
6062	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
6063	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
6064	pTlbe->fFlagsAndPhysRev &= ~fTlbAccessedDirty;
6065	}
6066
6067	/*
6068	* Look up the physical page info if necessary.
6069	*/
6070	uint8_t *pbMem = NULL;
6071	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6072	# ifdef IN_RING3
6073	pbMem = pTlbe->pbMappingR3;
6074	# else
6075	pbMem = NULL;
6076	# endif
6077	else
6078	{
6079	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
6080	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
6081	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
6082	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
6083	if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
6084	{ /* likely */ }
6085	else
6086	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
6087	pTlbe->pbMappingR3 = NULL;
6088	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
6089	\| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_UNASSIGNED);
6090	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
6091	&pbMem, &pTlbe->fFlagsAndPhysRev);
6092	AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
6093	# ifdef IN_RING3
6094	pTlbe->pbMappingR3 = pbMem;
6095	# endif
6096	}
6097
6098	/*
6099	* Check the physical page level access and mapping.
6100	*/
6101	if ( !(pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ))
6102	\|\| !(pTlbe->fFlagsAndPhysRev & ( (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PG_NO_WRITE : 0)
6103	\| (fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0))) )
6104	{ /* probably likely */ }
6105	else
6106	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem,
6107	pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
6108	pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
6109	: pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL
6110	: VERR_PGM_PHYS_TLB_CATCH_WRITE);
6111	Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
6112
6113	if (pbMem)
6114	{
6115	Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
6116	pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6117	fAccess \|= IEM_ACCESS_NOT_LOCKED;
6118	}
6119	else
6120	{
6121	Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
6122	RTGCPHYS const GCPhysFirst = pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6123	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6124	if (rcStrict != VINF_SUCCESS)
6125	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6126	}
6127
6128	void * const pvMem = pbMem;
6129
6130	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6131	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6132	if (fAccess & IEM_ACCESS_TYPE_READ)
6133	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6134
6135	#else /* !IEM_WITH_DATA_TLB */
6136
6137	RTGCPHYS GCPhysFirst;
6138	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
6139	if (rcStrict != VINF_SUCCESS)
6140	return rcStrict;
6141
6142	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6143	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6144	if (fAccess & IEM_ACCESS_TYPE_READ)
6145	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6146
6147	void *pvMem;
6148	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6149	if (rcStrict != VINF_SUCCESS)
6150	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6151
6152	#endif /* !IEM_WITH_DATA_TLB */
6153
6154	/*
6155	* Fill in the mapping table entry.
6156	*/
6157	pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
6158	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
6159	pVCpu->iem.s.iNextMapping = iMemMap + 1;
6160	pVCpu->iem.s.cActiveMappings += 1;
6161
6162	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6163	*ppvMem = pvMem;
6164
6165	return VINF_SUCCESS;
6166	}
6167
6168
6169	/**
6170	* Commits the guest memory if bounce buffered and unmaps it.
6171	*
6172	* @returns Strict VBox status code.
6173	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6174	* @param pvMem The mapping.
6175	* @param fAccess The kind of access.
6176	*/
6177	VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6178	{
6179	int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6180	AssertReturn(iMemMap >= 0, iMemMap);
6181
6182	/* If it's bounce buffered, we may need to write back the buffer. */
6183	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6184	{
6185	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6186	return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /fPostponeFail/);
6187	}
6188	/* Otherwise unlock it. */
6189	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6190	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6191
6192	/* Free the entry. */
6193	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6194	Assert(pVCpu->iem.s.cActiveMappings != 0);
6195	pVCpu->iem.s.cActiveMappings--;
6196	return VINF_SUCCESS;
6197	}
6198
6199	#ifdef IEM_WITH_SETJMP
6200
6201	/**
6202	* Maps the specified guest memory for the given kind of access, longjmp on
6203	* error.
6204	*
6205	* This may be using bounce buffering of the memory if it's crossing a page
6206	* boundary or if there is an access handler installed for any of it. Because
6207	* of lock prefix guarantees, we're in for some extra clutter when this
6208	* happens.
6209	*
6210	* This may raise a \#GP, \#SS, \#PF or \#AC.
6211	*
6212	* @returns Pointer to the mapped memory.
6213	*
6214	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6215	* @param cbMem The number of bytes to map. This is usually 1,
6216	* 2, 4, 6, 8, 12, 16, 32 or 512. When used by
6217	* string operations it can be up to a page.
6218	* @param iSegReg The index of the segment register to use for
6219	* this access. The base and limits are checked.
6220	* Use UINT8_MAX to indicate that no segmentation
6221	* is required (for IDT, GDT and LDT accesses).
6222	* @param GCPtrMem The address of the guest memory.
6223	* @param fAccess How the memory is being accessed. The
6224	* IEM_ACCESS_TYPE_XXX bit is used to figure out
6225	* how to map the memory, while the
6226	* IEM_ACCESS_WHAT_XXX bit is used when raising
6227	* exceptions.
6228	* @param uAlignCtl Alignment control:
6229	* - Bits 15:0 is the alignment mask.
6230	* - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
6231	* IEM_MEMMAP_F_ALIGN_SSE, and
6232	* IEM_MEMMAP_F_ALIGN_GP_OR_AC.
6233	* Pass zero to skip alignment.
6234	*/
6235	void *iemMemMapJmp(PVMCPUCC pVCpu, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess,
6236	uint32_t uAlignCtl) RT_NOEXCEPT
6237	{
6238	/*
6239	* Check the input, check segment access and adjust address
6240	* with segment base.
6241	*/
6242	Assert(cbMem <= 64 \|\| cbMem == 512 \|\| cbMem == 108 \|\| cbMem == 104 \|\| cbMem == 94); /* 512 is the max! */
6243	Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK)));
6244	Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
6245
6246	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
6247	if (rcStrict == VINF_SUCCESS) { /likely/ }
6248	else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6249
6250	/*
6251	* Alignment check.
6252	*/
6253	if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
6254	{ /* likelyish */ }
6255	else
6256	{
6257	/* Misaligned access. */
6258	if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6259	{
6260	if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
6261	\|\| ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
6262	&& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
6263	{
6264	AssertCompile(X86_CR0_AM == X86_EFL_AC);
6265
6266	if (iemMemAreAlignmentChecksEnabled(pVCpu))
6267	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6268	}
6269	else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
6270	&& iemMemAreAlignmentChecksEnabled(pVCpu)
6271	/** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
6272	* implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. */
6273	)
6274	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6275	else
6276	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
6277	}
6278	}
6279
6280	/*
6281	* Figure out which mapping entry to use.
6282	*/
6283	unsigned iMemMap = pVCpu->iem.s.iNextMapping;
6284	if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
6285	\|\| pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
6286	{
6287	iMemMap = iemMemMapFindFree(pVCpu);
6288	AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
6289	("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
6290	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
6291	pVCpu->iem.s.aMemMappings[2].fAccess),
6292	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_9));
6293	}
6294
6295	/*
6296	* Crossing a page boundary?
6297	*/
6298	if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE)
6299	{ /* No (likely). */ }
6300	else
6301	{
6302	void *pvMem;
6303	rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, cbMem, GCPtrMem, fAccess);
6304	if (rcStrict == VINF_SUCCESS)
6305	return pvMem;
6306	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6307	}
6308
6309	#ifdef IEM_WITH_DATA_TLB
6310	Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
6311
6312	/*
6313	* Get the TLB entry for this page.
6314	*/
6315	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem);
6316	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6317	if (pTlbe->uTag == uTag)
6318	STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
6319	else
6320	{
6321	pVCpu->iem.s.DataTlb.cTlbMisses++;
6322	PGMPTWALK Walk;
6323	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
6324	if (RT_FAILURE(rc))
6325	{
6326	Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
6327	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6328	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6329	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
6330	# endif
6331	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, rc);
6332	}
6333
6334	Assert(Walk.fSucceeded);
6335	pTlbe->uTag = uTag;
6336	pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A); /* skipping NX */
6337	pTlbe->GCPhys = Walk.GCPhys;
6338	pTlbe->pbMappingR3 = NULL;
6339	}
6340
6341	/*
6342	* Check the flags and physical revision.
6343	*/
6344	/** @todo make the caller pass these in with fAccess. */
6345	uint64_t const fNoUser = (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS && pVCpu->iem.s.uCpl == 3
6346	? IEMTLBE_F_PT_NO_USER : 0;
6347	uint64_t const fNoWriteNoDirty = fAccess & IEM_ACCESS_TYPE_WRITE
6348	? IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PT_NO_DIRTY
6349	\| ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)
6350	\|\| (pVCpu->iem.s.uCpl == 3 && (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6351	? IEMTLBE_F_PT_NO_WRITE : 0)
6352	: 0;
6353	uint64_t const fNoRead = fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0;
6354	uint8_t *pbMem = NULL;
6355	if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PT_NO_ACCESSED \| fNoRead \| fNoWriteNoDirty \| fNoUser))
6356	== pVCpu->iem.s.DataTlb.uTlbPhysRev)
6357	# ifdef IN_RING3
6358	pbMem = pTlbe->pbMappingR3;
6359	# else
6360	pbMem = NULL;
6361	# endif
6362	else
6363	{
6364	/*
6365	* Okay, something isn't quite right or needs refreshing.
6366	*/
6367	/* Write to read only memory? */
6368	if (pTlbe->fFlagsAndPhysRev & fNoWriteNoDirty & IEMTLBE_F_PT_NO_WRITE)
6369	{
6370	Log(("iemMemMapJmp: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6371	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6372	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6373	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6374	# endif
6375	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6376	}
6377
6378	/* Kernel memory accessed by userland? */
6379	if (pTlbe->fFlagsAndPhysRev & fNoUser & IEMTLBE_F_PT_NO_USER)
6380	{
6381	Log(("iemMemMapJmp: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6382	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6383	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6384	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6385	# endif
6386	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
6387	}
6388
6389	/* Set the dirty / access flags.
6390	ASSUMES this is set when the address is translated rather than on commit... */
6391	/** @todo testcase: check when A and D bits are actually set by the CPU. */
6392	if (pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) \| IEMTLBE_F_PT_NO_ACCESSED))
6393	{
6394	uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
6395	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6396	AssertRC(rc2);
6397	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
6398	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
6399	pTlbe->fFlagsAndPhysRev &= ~((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) \| IEMTLBE_F_PT_NO_ACCESSED);
6400	}
6401
6402	/*
6403	* Check if the physical page info needs updating.
6404	*/
6405	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6406	# ifdef IN_RING3
6407	pbMem = pTlbe->pbMappingR3;
6408	# else
6409	pbMem = NULL;
6410	# endif
6411	else
6412	{
6413	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
6414	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
6415	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
6416	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
6417	pTlbe->pbMappingR3 = NULL;
6418	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
6419	\| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_UNASSIGNED);
6420	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
6421	&pbMem, &pTlbe->fFlagsAndPhysRev);
6422	AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
6423	# ifdef IN_RING3
6424	pTlbe->pbMappingR3 = pbMem;
6425	# endif
6426	}
6427
6428	/*
6429	* Check the physical page level access and mapping.
6430	*/
6431	if (!(pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty \| fNoRead) & (IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ))))
6432	{ /* probably likely */ }
6433	else
6434	{
6435	rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, (void **)&pbMem, cbMem,
6436	pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
6437	pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
6438	: pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL
6439	: VERR_PGM_PHYS_TLB_CATCH_WRITE);
6440	if (rcStrict == VINF_SUCCESS)
6441	return pbMem;
6442	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6443	}
6444	}
6445	Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
6446
6447	if (pbMem)
6448	{
6449	Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
6450	pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6451	fAccess \|= IEM_ACCESS_NOT_LOCKED;
6452	}
6453	else
6454	{
6455	Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
6456	RTGCPHYS const GCPhysFirst = pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6457	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6458	if (rcStrict == VINF_SUCCESS)
6459	return pbMem;
6460	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6461	}
6462
6463	void * const pvMem = pbMem;
6464
6465	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6466	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6467	if (fAccess & IEM_ACCESS_TYPE_READ)
6468	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6469
6470	#else /* !IEM_WITH_DATA_TLB */
6471
6472
6473	RTGCPHYS GCPhysFirst;
6474	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
6475	if (rcStrict == VINF_SUCCESS) { /likely/ }
6476	else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6477
6478	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6479	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6480	if (fAccess & IEM_ACCESS_TYPE_READ)
6481	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6482
6483	void *pvMem;
6484	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6485	if (rcStrict == VINF_SUCCESS)
6486	{ /* likely */ }
6487	else
6488	{
6489	rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6490	if (rcStrict == VINF_SUCCESS)
6491	return pvMem;
6492	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6493	}
6494
6495	#endif /* !IEM_WITH_DATA_TLB */
6496
6497	/*
6498	* Fill in the mapping table entry.
6499	*/
6500	pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
6501	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
6502	pVCpu->iem.s.iNextMapping = iMemMap + 1;
6503	pVCpu->iem.s.cActiveMappings++;
6504
6505	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6506	return pvMem;
6507	}
6508
6509
6510	/**
6511	* Commits the guest memory if bounce buffered and unmaps it, longjmp on error.
6512	*
6513	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6514	* @param pvMem The mapping.
6515	* @param fAccess The kind of access.
6516	*/
6517	void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6518	{
6519	int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6520	AssertStmt(iMemMap >= 0, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), iMemMap));
6521
6522	/* If it's bounce buffered, we may need to write back the buffer. */
6523	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6524	{
6525	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6526	{
6527	VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /fPostponeFail/);
6528	if (rcStrict == VINF_SUCCESS)
6529	return;
6530	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6531	}
6532	}
6533	/* Otherwise unlock it. */
6534	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6535	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6536
6537	/* Free the entry. */
6538	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6539	Assert(pVCpu->iem.s.cActiveMappings != 0);
6540	pVCpu->iem.s.cActiveMappings--;
6541	}
6542
6543	#endif /* IEM_WITH_SETJMP */
6544
6545	#ifndef IN_RING3
6546	/**
6547	* Commits the guest memory if bounce buffered and unmaps it, if any bounce
6548	* buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).
6549	*
6550	* Allows the instruction to be completed and retired, while the IEM user will
6551	* return to ring-3 immediately afterwards and do the postponed writes there.
6552	*
6553	* @returns VBox status code (no strict statuses). Caller must check
6554	* VMCPU_FF_IEM before repeating string instructions and similar stuff.
6555	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6556	* @param pvMem The mapping.
6557	* @param fAccess The kind of access.
6558	*/
6559	VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6560	{
6561	int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6562	AssertReturn(iMemMap >= 0, iMemMap);
6563
6564	/* If it's bounce buffered, we may need to write back the buffer. */
6565	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6566	{
6567	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6568	return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /fPostponeFail/);
6569	}
6570	/* Otherwise unlock it. */
6571	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6572	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6573
6574	/* Free the entry. */
6575	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6576	Assert(pVCpu->iem.s.cActiveMappings != 0);
6577	pVCpu->iem.s.cActiveMappings--;
6578	return VINF_SUCCESS;
6579	}
6580	#endif
6581
6582
6583	/**
6584	* Rollbacks mappings, releasing page locks and such.
6585	*
6586	* The caller shall only call this after checking cActiveMappings.
6587	*
6588	* @returns Strict VBox status code to pass up.
6589	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6590	*/
6591	void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT
6592	{
6593	Assert(pVCpu->iem.s.cActiveMappings > 0);
6594
6595	uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
6596	while (iMemMap-- > 0)
6597	{
6598	uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;
6599	if (fAccess != IEM_ACCESS_INVALID)
6600	{
6601	AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));
6602	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6603	if (!(fAccess & (IEM_ACCESS_BOUNCE_BUFFERED \| IEM_ACCESS_NOT_LOCKED)))
6604	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6605	AssertMsg(pVCpu->iem.s.cActiveMappings > 0,
6606	("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",
6607	iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,
6608	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));
6609	pVCpu->iem.s.cActiveMappings--;
6610	}
6611	}
6612	}
6613
6614
6615	/**
6616	* Fetches a data byte.
6617	*
6618	* @returns Strict VBox status code.
6619	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6620	* @param pu8Dst Where to return the byte.
6621	* @param iSegReg The index of the segment register to use for
6622	* this access. The base and limits are checked.
6623	* @param GCPtrMem The address of the guest memory.
6624	*/
6625	VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6626	{
6627	/* The lazy approach for now... */
6628	uint8_t const *pu8Src;
6629	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu8Src, sizeof(pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0);
6630	if (rc == VINF_SUCCESS)
6631	{
6632	pu8Dst = pu8Src;
6633	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
6634	}
6635	return rc;
6636	}
6637
6638
6639	#ifdef IEM_WITH_SETJMP
6640	/**
6641	* Fetches a data byte, longjmp on error.
6642	*
6643	* @returns The byte.
6644	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6645	* @param iSegReg The index of the segment register to use for
6646	* this access. The base and limits are checked.
6647	* @param GCPtrMem The address of the guest memory.
6648	*/
6649	uint8_t iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6650	{
6651	/* The lazy approach for now... */
6652	uint8_t const pu8Src = (uint8_t const )iemMemMapJmp(pVCpu, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0);
6653	uint8_t const bRet = *pu8Src;
6654	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
6655	return bRet;
6656	}
6657	#endif /* IEM_WITH_SETJMP */
6658
6659
6660	/**
6661	* Fetches a data word.
6662	*
6663	* @returns Strict VBox status code.
6664	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6665	* @param pu16Dst Where to return the word.
6666	* @param iSegReg The index of the segment register to use for
6667	* this access. The base and limits are checked.
6668	* @param GCPtrMem The address of the guest memory.
6669	*/
6670	VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6671	{
6672	/* The lazy approach for now... */
6673	uint16_t const *pu16Src;
6674	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem,
6675	IEM_ACCESS_DATA_R, sizeof(*pu16Src) - 1);
6676	if (rc == VINF_SUCCESS)
6677	{
6678	pu16Dst = pu16Src;
6679	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
6680	}
6681	return rc;
6682	}
6683
6684
6685	#ifdef IEM_WITH_SETJMP
6686	/**
6687	* Fetches a data word, longjmp on error.
6688	*
6689	* @returns The word
6690	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6691	* @param iSegReg The index of the segment register to use for
6692	* this access. The base and limits are checked.
6693	* @param GCPtrMem The address of the guest memory.
6694	*/
6695	uint16_t iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6696	{
6697	/* The lazy approach for now... */
6698	uint16_t const pu16Src = (uint16_t const )iemMemMapJmp(pVCpu, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6699	sizeof(*pu16Src) - 1);
6700	uint16_t const u16Ret = *pu16Src;
6701	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
6702	return u16Ret;
6703	}
6704	#endif
6705
6706
6707	/**
6708	* Fetches a data dword.
6709	*
6710	* @returns Strict VBox status code.
6711	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6712	* @param pu32Dst Where to return the dword.
6713	* @param iSegReg The index of the segment register to use for
6714	* this access. The base and limits are checked.
6715	* @param GCPtrMem The address of the guest memory.
6716	*/
6717	VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6718	{
6719	/* The lazy approach for now... */
6720	uint32_t const *pu32Src;
6721	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem,
6722	IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6723	if (rc == VINF_SUCCESS)
6724	{
6725	pu32Dst = pu32Src;
6726	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6727	}
6728	return rc;
6729	}
6730
6731
6732	/**
6733	* Fetches a data dword and zero extends it to a qword.
6734	*
6735	* @returns Strict VBox status code.
6736	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6737	* @param pu64Dst Where to return the qword.
6738	* @param iSegReg The index of the segment register to use for
6739	* this access. The base and limits are checked.
6740	* @param GCPtrMem The address of the guest memory.
6741	*/
6742	VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6743	{
6744	/* The lazy approach for now... */
6745	uint32_t const *pu32Src;
6746	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem,
6747	IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6748	if (rc == VINF_SUCCESS)
6749	{
6750	pu64Dst = pu32Src;
6751	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6752	}
6753	return rc;
6754	}
6755
6756
6757	#ifdef IEM_WITH_SETJMP
6758
6759	/**
6760	* Fetches a data dword, longjmp on error, fallback/safe version.
6761	*
6762	* @returns The dword
6763	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6764	* @param iSegReg The index of the segment register to use for
6765	* this access. The base and limits are checked.
6766	* @param GCPtrMem The address of the guest memory.
6767	*/
6768	uint32_t iemMemFetchDataU32SafeJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6769	{
6770	uint32_t const pu32Src = (uint32_t const )iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6771	sizeof(*pu32Src) - 1);
6772	uint32_t const u32Ret = *pu32Src;
6773	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6774	return u32Ret;
6775	}
6776
6777
6778	/**
6779	* Fetches a data dword, longjmp on error.
6780	*
6781	* @returns The dword
6782	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6783	* @param iSegReg The index of the segment register to use for
6784	* this access. The base and limits are checked.
6785	* @param GCPtrMem The address of the guest memory.
6786	*/
6787	uint32_t iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6788	{
6789	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
6790	/*
6791	* Convert from segmented to flat address and check that it doesn't cross a page boundrary.
6792	*/
6793	RTGCPTR GCPtrEff = iemMemApplySegmentToReadJmp(pVCpu, iSegReg, sizeof(uint32_t), GCPtrMem);
6794	if (RT_LIKELY((GCPtrEff & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(uint32_t)))
6795	{
6796	/*
6797	* TLB lookup.
6798	*/
6799	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrEff);
6800	PIEMTLBENTRY pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6801	if (pTlbe->uTag == uTag)
6802	{
6803	/*
6804	* Check TLB page table level access flags.
6805	*/
6806	uint64_t const fNoUser = pVCpu->iem.s.uCpl == 3 ? IEMTLBE_F_PT_NO_USER : 0;
6807	if ( (pTlbe->fFlagsAndPhysRev & ( IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PG_NO_READ
6808	\| IEMTLBE_F_PT_NO_ACCESSED \| IEMTLBE_F_NO_MAPPINGR3 \| fNoUser))
6809	== pVCpu->iem.s.DataTlb.uTlbPhysRev)
6810	{
6811	STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
6812
6813	/*
6814	* Alignment check:
6815	*/
6816	/** @todo check priority \#AC vs \#PF */
6817	if ( !(GCPtrEff & (sizeof(uint32_t) - 1))
6818	\|\| !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
6819	\|\| !pVCpu->cpum.GstCtx.eflags.Bits.u1AC
6820	\|\| pVCpu->iem.s.uCpl != 3)
6821	{
6822	/*
6823	* Fetch and return the dword
6824	*/
6825	Assert(pTlbe->pbMappingR3); /* (Only ever cleared by the owning EMT.) */
6826	Assert(!((uintptr_t)pTlbe->pbMappingR3 & GUEST_PAGE_OFFSET_MASK));
6827	return (uint32_t const )&pTlbe->pbMappingR3[GCPtrEff & GUEST_PAGE_OFFSET_MASK];
6828	}
6829	Log10(("iemMemFetchDataU32Jmp: Raising #AC for %RGv\n", GCPtrEff));
6830	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6831	}
6832	}
6833	}
6834
6835	/* Fall back on the slow careful approach in case of TLB miss, MMIO, exception
6836	outdated page pointer, or other troubles. */
6837	Log10(("iemMemFetchDataU32Jmp: %u:%RGv fallback\n", iSegReg, GCPtrMem));
6838	return iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
6839
6840	# else
6841	uint32_t const pu32Src = (uint32_t const )iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem,
6842	IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6843	uint32_t const u32Ret = *pu32Src;
6844	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6845	return u32Ret;
6846	# endif
6847	}
6848	#endif
6849
6850
6851	#ifdef SOME_UNUSED_FUNCTION
6852	/**
6853	* Fetches a data dword and sign extends it to a qword.
6854	*
6855	* @returns Strict VBox status code.
6856	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6857	* @param pu64Dst Where to return the sign extended value.
6858	* @param iSegReg The index of the segment register to use for
6859	* this access. The base and limits are checked.
6860	* @param GCPtrMem The address of the guest memory.
6861	*/
6862	VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6863	{
6864	/* The lazy approach for now... */
6865	int32_t const *pi32Src;
6866	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pi32Src, sizeof(pi32Src), iSegReg, GCPtrMem,
6867	IEM_ACCESS_DATA_R, sizeof(*pi32Src) - 1);
6868	if (rc == VINF_SUCCESS)
6869	{
6870	pu64Dst = pi32Src;
6871	rc = iemMemCommitAndUnmap(pVCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
6872	}
6873	#ifdef __GNUC__ /* warning: GCC may be a royal pain */
6874	else
6875	*pu64Dst = 0;
6876	#endif
6877	return rc;
6878	}
6879	#endif
6880
6881
6882	/**
6883	* Fetches a data qword.
6884	*
6885	* @returns Strict VBox status code.
6886	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6887	* @param pu64Dst Where to return the qword.
6888	* @param iSegReg The index of the segment register to use for
6889	* this access. The base and limits are checked.
6890	* @param GCPtrMem The address of the guest memory.
6891	*/
6892	VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6893	{
6894	/* The lazy approach for now... */
6895	uint64_t const *pu64Src;
6896	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem,
6897	IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1);
6898	if (rc == VINF_SUCCESS)
6899	{
6900	pu64Dst = pu64Src;
6901	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6902	}
6903	return rc;
6904	}
6905
6906
6907	#ifdef IEM_WITH_SETJMP
6908	/**
6909	* Fetches a data qword, longjmp on error.
6910	*
6911	* @returns The qword.
6912	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6913	* @param iSegReg The index of the segment register to use for
6914	* this access. The base and limits are checked.
6915	* @param GCPtrMem The address of the guest memory.
6916	*/
6917	uint64_t iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6918	{
6919	/* The lazy approach for now... */
6920	uint64_t const pu64Src = (uint64_t const )iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem,
6921	IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1);
6922	uint64_t const u64Ret = *pu64Src;
6923	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6924	return u64Ret;
6925	}
6926	#endif
6927
6928
6929	/**
6930	* Fetches a data qword, aligned at a 16 byte boundrary (for SSE).
6931	*
6932	* @returns Strict VBox status code.
6933	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6934	* @param pu64Dst Where to return the qword.
6935	* @param iSegReg The index of the segment register to use for
6936	* this access. The base and limits are checked.
6937	* @param GCPtrMem The address of the guest memory.
6938	*/
6939	VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6940	{
6941	/* The lazy approach for now... */
6942	uint64_t const *pu64Src;
6943	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem,
6944	IEM_ACCESS_DATA_R, 15 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
6945	if (rc == VINF_SUCCESS)
6946	{
6947	pu64Dst = pu64Src;
6948	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6949	}
6950	return rc;
6951	}
6952
6953
6954	#ifdef IEM_WITH_SETJMP
6955	/**
6956	* Fetches a data qword, longjmp on error.
6957	*
6958	* @returns The qword.
6959	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6960	* @param iSegReg The index of the segment register to use for
6961	* this access. The base and limits are checked.
6962	* @param GCPtrMem The address of the guest memory.
6963	*/
6964	uint64_t iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6965	{
6966	/* The lazy approach for now... */
6967	uint64_t const pu64Src = (uint64_t const )iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6968	15 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
6969	uint64_t const u64Ret = *pu64Src;
6970	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6971	return u64Ret;
6972	}
6973	#endif
6974
6975
6976	/**
6977	* Fetches a data tword.
6978	*
6979	* @returns Strict VBox status code.
6980	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6981	* @param pr80Dst Where to return the tword.
6982	* @param iSegReg The index of the segment register to use for
6983	* this access. The base and limits are checked.
6984	* @param GCPtrMem The address of the guest memory.
6985	*/
6986	VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6987	{
6988	/* The lazy approach for now... */
6989	PCRTFLOAT80U pr80Src;
6990	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pr80Src, sizeof(pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7);
6991	if (rc == VINF_SUCCESS)
6992	{
6993	pr80Dst = pr80Src;
6994	rc = iemMemCommitAndUnmap(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
6995	}
6996	return rc;
6997	}
6998
6999
7000	#ifdef IEM_WITH_SETJMP
7001	/**
7002	* Fetches a data tword, longjmp on error.
7003	*
7004	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7005	* @param pr80Dst Where to return the tword.
7006	* @param iSegReg The index of the segment register to use for
7007	* this access. The base and limits are checked.
7008	* @param GCPtrMem The address of the guest memory.
7009	*/
7010	void iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7011	{
7012	/* The lazy approach for now... */
7013	PCRTFLOAT80U pr80Src = (PCRTFLOAT80U)iemMemMapJmp(pVCpu, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7);
7014	pr80Dst = pr80Src;
7015	iemMemCommitAndUnmapJmp(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
7016	}
7017	#endif
7018
7019
7020	/**
7021	* Fetches a data decimal tword.
7022	*
7023	* @returns Strict VBox status code.
7024	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7025	* @param pd80Dst Where to return the tword.
7026	* @param iSegReg The index of the segment register to use for
7027	* this access. The base and limits are checked.
7028	* @param GCPtrMem The address of the guest memory.
7029	*/
7030	VBOXSTRICTRC iemMemFetchDataD80(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7031	{
7032	/* The lazy approach for now... */
7033	PCRTPBCD80U pd80Src;
7034	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pd80Src, sizeof(pd80Src), iSegReg, GCPtrMem,
7035	IEM_ACCESS_DATA_R, 7 /** @todo FBLD alignment check */);
7036	if (rc == VINF_SUCCESS)
7037	{
7038	pd80Dst = pd80Src;
7039	rc = iemMemCommitAndUnmap(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R);
7040	}
7041	return rc;
7042	}
7043
7044
7045	#ifdef IEM_WITH_SETJMP
7046	/**
7047	* Fetches a data decimal tword, longjmp on error.
7048	*
7049	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7050	* @param pd80Dst Where to return the tword.
7051	* @param iSegReg The index of the segment register to use for
7052	* this access. The base and limits are checked.
7053	* @param GCPtrMem The address of the guest memory.
7054	*/
7055	void iemMemFetchDataD80Jmp(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7056	{
7057	/* The lazy approach for now... */
7058	PCRTPBCD80U pd80Src = (PCRTPBCD80U)iemMemMapJmp(pVCpu, sizeof(*pd80Src), iSegReg, GCPtrMem,
7059	IEM_ACCESS_DATA_R, 7 /** @todo FBSTP alignment check */);
7060	pd80Dst = pd80Src;
7061	iemMemCommitAndUnmapJmp(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R);
7062	}
7063	#endif
7064
7065
7066	/**
7067	* Fetches a data dqword (double qword), generally SSE related.
7068	*
7069	* @returns Strict VBox status code.
7070	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7071	* @param pu128Dst Where to return the qword.
7072	* @param iSegReg The index of the segment register to use for
7073	* this access. The base and limits are checked.
7074	* @param GCPtrMem The address of the guest memory.
7075	*/
7076	VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7077	{
7078	/* The lazy approach for now... */
7079	PCRTUINT128U pu128Src;
7080	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Src, sizeof(pu128Src), iSegReg, GCPtrMem,
7081	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7082	if (rc == VINF_SUCCESS)
7083	{
7084	pu128Dst->au64[0] = pu128Src->au64[0];
7085	pu128Dst->au64[1] = pu128Src->au64[1];
7086	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7087	}
7088	return rc;
7089	}
7090
7091
7092	#ifdef IEM_WITH_SETJMP
7093	/**
7094	* Fetches a data dqword (double qword), generally SSE related.
7095	*
7096	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7097	* @param pu128Dst Where to return the qword.
7098	* @param iSegReg The index of the segment register to use for
7099	* this access. The base and limits are checked.
7100	* @param GCPtrMem The address of the guest memory.
7101	*/
7102	void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7103	{
7104	/* The lazy approach for now... */
7105	PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem,
7106	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7107	pu128Dst->au64[0] = pu128Src->au64[0];
7108	pu128Dst->au64[1] = pu128Src->au64[1];
7109	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7110	}
7111	#endif
7112
7113
7114	/**
7115	* Fetches a data dqword (double qword) at an aligned address, generally SSE
7116	* related.
7117	*
7118	* Raises \#GP(0) if not aligned.
7119	*
7120	* @returns Strict VBox status code.
7121	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7122	* @param pu128Dst Where to return the qword.
7123	* @param iSegReg The index of the segment register to use for
7124	* this access. The base and limits are checked.
7125	* @param GCPtrMem The address of the guest memory.
7126	*/
7127	VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7128	{
7129	/* The lazy approach for now... */
7130	PCRTUINT128U pu128Src;
7131	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Src, sizeof(pu128Src), iSegReg, GCPtrMem,
7132	IEM_ACCESS_DATA_R, (sizeof(*pu128Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7133	if (rc == VINF_SUCCESS)
7134	{
7135	pu128Dst->au64[0] = pu128Src->au64[0];
7136	pu128Dst->au64[1] = pu128Src->au64[1];
7137	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7138	}
7139	return rc;
7140	}
7141
7142
7143	#ifdef IEM_WITH_SETJMP
7144	/**
7145	* Fetches a data dqword (double qword) at an aligned address, generally SSE
7146	* related, longjmp on error.
7147	*
7148	* Raises \#GP(0) if not aligned.
7149	*
7150	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7151	* @param pu128Dst Where to return the qword.
7152	* @param iSegReg The index of the segment register to use for
7153	* this access. The base and limits are checked.
7154	* @param GCPtrMem The address of the guest memory.
7155	*/
7156	void iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7157	{
7158	/* The lazy approach for now... */
7159	PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
7160	(sizeof(*pu128Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7161	pu128Dst->au64[0] = pu128Src->au64[0];
7162	pu128Dst->au64[1] = pu128Src->au64[1];
7163	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7164	}
7165	#endif
7166
7167
7168	/**
7169	* Fetches a data oword (octo word), generally AVX related.
7170	*
7171	* @returns Strict VBox status code.
7172	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7173	* @param pu256Dst Where to return the qword.
7174	* @param iSegReg The index of the segment register to use for
7175	* this access. The base and limits are checked.
7176	* @param GCPtrMem The address of the guest memory.
7177	*/
7178	VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7179	{
7180	/* The lazy approach for now... */
7181	PCRTUINT256U pu256Src;
7182	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Src, sizeof(pu256Src), iSegReg, GCPtrMem,
7183	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7184	if (rc == VINF_SUCCESS)
7185	{
7186	pu256Dst->au64[0] = pu256Src->au64[0];
7187	pu256Dst->au64[1] = pu256Src->au64[1];
7188	pu256Dst->au64[2] = pu256Src->au64[2];
7189	pu256Dst->au64[3] = pu256Src->au64[3];
7190	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7191	}
7192	return rc;
7193	}
7194
7195
7196	#ifdef IEM_WITH_SETJMP
7197	/**
7198	* Fetches a data oword (octo word), generally AVX related.
7199	*
7200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7201	* @param pu256Dst Where to return the qword.
7202	* @param iSegReg The index of the segment register to use for
7203	* this access. The base and limits are checked.
7204	* @param GCPtrMem The address of the guest memory.
7205	*/
7206	void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7207	{
7208	/* The lazy approach for now... */
7209	PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem,
7210	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7211	pu256Dst->au64[0] = pu256Src->au64[0];
7212	pu256Dst->au64[1] = pu256Src->au64[1];
7213	pu256Dst->au64[2] = pu256Src->au64[2];
7214	pu256Dst->au64[3] = pu256Src->au64[3];
7215	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7216	}
7217	#endif
7218
7219
7220	/**
7221	* Fetches a data oword (octo word) at an aligned address, generally AVX
7222	* related.
7223	*
7224	* Raises \#GP(0) if not aligned.
7225	*
7226	* @returns Strict VBox status code.
7227	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7228	* @param pu256Dst Where to return the qword.
7229	* @param iSegReg The index of the segment register to use for
7230	* this access. The base and limits are checked.
7231	* @param GCPtrMem The address of the guest memory.
7232	*/
7233	VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7234	{
7235	/* The lazy approach for now... */
7236	PCRTUINT256U pu256Src;
7237	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Src, sizeof(pu256Src), iSegReg, GCPtrMem,
7238	IEM_ACCESS_DATA_R, (sizeof(*pu256Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7239	if (rc == VINF_SUCCESS)
7240	{
7241	pu256Dst->au64[0] = pu256Src->au64[0];
7242	pu256Dst->au64[1] = pu256Src->au64[1];
7243	pu256Dst->au64[2] = pu256Src->au64[2];
7244	pu256Dst->au64[3] = pu256Src->au64[3];
7245	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7246	}
7247	return rc;
7248	}
7249
7250
7251	#ifdef IEM_WITH_SETJMP
7252	/**
7253	* Fetches a data oword (octo word) at an aligned address, generally AVX
7254	* related, longjmp on error.
7255	*
7256	* Raises \#GP(0) if not aligned.
7257	*
7258	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7259	* @param pu256Dst Where to return the qword.
7260	* @param iSegReg The index of the segment register to use for
7261	* this access. The base and limits are checked.
7262	* @param GCPtrMem The address of the guest memory.
7263	*/
7264	void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7265	{
7266	/* The lazy approach for now... */
7267	PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
7268	(sizeof(*pu256Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7269	pu256Dst->au64[0] = pu256Src->au64[0];
7270	pu256Dst->au64[1] = pu256Src->au64[1];
7271	pu256Dst->au64[2] = pu256Src->au64[2];
7272	pu256Dst->au64[3] = pu256Src->au64[3];
7273	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7274	}
7275	#endif
7276
7277
7278
7279	/**
7280	* Fetches a descriptor register (lgdt, lidt).
7281	*
7282	* @returns Strict VBox status code.
7283	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7284	* @param pcbLimit Where to return the limit.
7285	* @param pGCPtrBase Where to return the base.
7286	* @param iSegReg The index of the segment register to use for
7287	* this access. The base and limits are checked.
7288	* @param GCPtrMem The address of the guest memory.
7289	* @param enmOpSize The effective operand size.
7290	*/
7291	VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
7292	RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT
7293	{
7294	/*
7295	* Just like SIDT and SGDT, the LIDT and LGDT instructions are a
7296	* little special:
7297	* - The two reads are done separately.
7298	* - Operand size override works in 16-bit and 32-bit code, but 64-bit.
7299	* - We suspect the 386 to actually commit the limit before the base in
7300	* some cases (search for 386 in bs3CpuBasic2_lidt_lgdt_One). We
7301	* don't try emulate this eccentric behavior, because it's not well
7302	* enough understood and rather hard to trigger.
7303	* - The 486 seems to do a dword limit read when the operand size is 32-bit.
7304	*/
7305	VBOXSTRICTRC rcStrict;
7306	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
7307	{
7308	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7309	if (rcStrict == VINF_SUCCESS)
7310	rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2);
7311	}
7312	else
7313	{
7314	uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */
7315	if (enmOpSize == IEMMODE_32BIT)
7316	{
7317	if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486)
7318	{
7319	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7320	if (rcStrict == VINF_SUCCESS)
7321	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7322	}
7323	else
7324	{
7325	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem);
7326	if (rcStrict == VINF_SUCCESS)
7327	{
7328	*pcbLimit = (uint16_t)uTmp;
7329	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7330	}
7331	}
7332	if (rcStrict == VINF_SUCCESS)
7333	*pGCPtrBase = uTmp;
7334	}
7335	else
7336	{
7337	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7338	if (rcStrict == VINF_SUCCESS)
7339	{
7340	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7341	if (rcStrict == VINF_SUCCESS)
7342	*pGCPtrBase = uTmp & UINT32_C(0x00ffffff);
7343	}
7344	}
7345	}
7346	return rcStrict;
7347	}
7348
7349
7350
7351	/**
7352	* Stores a data byte.
7353	*
7354	* @returns Strict VBox status code.
7355	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7356	* @param iSegReg The index of the segment register to use for
7357	* this access. The base and limits are checked.
7358	* @param GCPtrMem The address of the guest memory.
7359	* @param u8Value The value to store.
7360	*/
7361	VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT
7362	{
7363	/* The lazy approach for now... */
7364	uint8_t *pu8Dst;
7365	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu8Dst, sizeof(pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0);
7366	if (rc == VINF_SUCCESS)
7367	{
7368	*pu8Dst = u8Value;
7369	rc = iemMemCommitAndUnmap(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
7370	}
7371	return rc;
7372	}
7373
7374
7375	#ifdef IEM_WITH_SETJMP
7376	/**
7377	* Stores a data byte, longjmp on error.
7378	*
7379	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7380	* @param iSegReg The index of the segment register to use for
7381	* this access. The base and limits are checked.
7382	* @param GCPtrMem The address of the guest memory.
7383	* @param u8Value The value to store.
7384	*/
7385	void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT
7386	{
7387	/* The lazy approach for now... */
7388	uint8_t pu8Dst = (uint8_t )iemMemMapJmp(pVCpu, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0);
7389	*pu8Dst = u8Value;
7390	iemMemCommitAndUnmapJmp(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
7391	}
7392	#endif
7393
7394
7395	/**
7396	* Stores a data word.
7397	*
7398	* @returns Strict VBox status code.
7399	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7400	* @param iSegReg The index of the segment register to use for
7401	* this access. The base and limits are checked.
7402	* @param GCPtrMem The address of the guest memory.
7403	* @param u16Value The value to store.
7404	*/
7405	VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT
7406	{
7407	/* The lazy approach for now... */
7408	uint16_t *pu16Dst;
7409	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Dst, sizeof(pu16Dst), iSegReg, GCPtrMem,
7410	IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1);
7411	if (rc == VINF_SUCCESS)
7412	{
7413	*pu16Dst = u16Value;
7414	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
7415	}
7416	return rc;
7417	}
7418
7419
7420	#ifdef IEM_WITH_SETJMP
7421	/**
7422	* Stores a data word, longjmp on error.
7423	*
7424	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7425	* @param iSegReg The index of the segment register to use for
7426	* this access. The base and limits are checked.
7427	* @param GCPtrMem The address of the guest memory.
7428	* @param u16Value The value to store.
7429	*/
7430	void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT
7431	{
7432	/* The lazy approach for now... */
7433	uint16_t pu16Dst = (uint16_t )iemMemMapJmp(pVCpu, sizeof(*pu16Dst), iSegReg, GCPtrMem,
7434	IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1);
7435	*pu16Dst = u16Value;
7436	iemMemCommitAndUnmapJmp(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
7437	}
7438	#endif
7439
7440
7441	/**
7442	* Stores a data dword.
7443	*
7444	* @returns Strict VBox status code.
7445	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7446	* @param iSegReg The index of the segment register to use for
7447	* this access. The base and limits are checked.
7448	* @param GCPtrMem The address of the guest memory.
7449	* @param u32Value The value to store.
7450	*/
7451	VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT
7452	{
7453	/* The lazy approach for now... */
7454	uint32_t *pu32Dst;
7455	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Dst, sizeof(pu32Dst), iSegReg, GCPtrMem,
7456	IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1);
7457	if (rc == VINF_SUCCESS)
7458	{
7459	*pu32Dst = u32Value;
7460	rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
7461	}
7462	return rc;
7463	}
7464
7465
7466	#ifdef IEM_WITH_SETJMP
7467	/**
7468	* Stores a data dword.
7469	*
7470	* @returns Strict VBox status code.
7471	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7472	* @param iSegReg The index of the segment register to use for
7473	* this access. The base and limits are checked.
7474	* @param GCPtrMem The address of the guest memory.
7475	* @param u32Value The value to store.
7476	*/
7477	void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT
7478	{
7479	/* The lazy approach for now... */
7480	uint32_t pu32Dst = (uint32_t )iemMemMapJmp(pVCpu, sizeof(*pu32Dst), iSegReg, GCPtrMem,
7481	IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1);
7482	*pu32Dst = u32Value;
7483	iemMemCommitAndUnmapJmp(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
7484	}
7485	#endif
7486
7487
7488	/**
7489	* Stores a data qword.
7490	*
7491	* @returns Strict VBox status code.
7492	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7493	* @param iSegReg The index of the segment register to use for
7494	* this access. The base and limits are checked.
7495	* @param GCPtrMem The address of the guest memory.
7496	* @param u64Value The value to store.
7497	*/
7498	VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT
7499	{
7500	/* The lazy approach for now... */
7501	uint64_t *pu64Dst;
7502	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Dst, sizeof(pu64Dst), iSegReg, GCPtrMem,
7503	IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1);
7504	if (rc == VINF_SUCCESS)
7505	{
7506	*pu64Dst = u64Value;
7507	rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
7508	}
7509	return rc;
7510	}
7511
7512
7513	#ifdef IEM_WITH_SETJMP
7514	/**
7515	* Stores a data qword, longjmp on error.
7516	*
7517	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7518	* @param iSegReg The index of the segment register to use for
7519	* this access. The base and limits are checked.
7520	* @param GCPtrMem The address of the guest memory.
7521	* @param u64Value The value to store.
7522	*/
7523	void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT
7524	{
7525	/* The lazy approach for now... */
7526	uint64_t pu64Dst = (uint64_t )iemMemMapJmp(pVCpu, sizeof(*pu64Dst), iSegReg, GCPtrMem,
7527	IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1);
7528	*pu64Dst = u64Value;
7529	iemMemCommitAndUnmapJmp(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
7530	}
7531	#endif
7532
7533
7534	/**
7535	* Stores a data dqword.
7536	*
7537	* @returns Strict VBox status code.
7538	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7539	* @param iSegReg The index of the segment register to use for
7540	* this access. The base and limits are checked.
7541	* @param GCPtrMem The address of the guest memory.
7542	* @param u128Value The value to store.
7543	*/
7544	VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7545	{
7546	/* The lazy approach for now... */
7547	PRTUINT128U pu128Dst;
7548	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Dst, sizeof(pu128Dst), iSegReg, GCPtrMem,
7549	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7550	if (rc == VINF_SUCCESS)
7551	{
7552	pu128Dst->au64[0] = u128Value.au64[0];
7553	pu128Dst->au64[1] = u128Value.au64[1];
7554	rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7555	}
7556	return rc;
7557	}
7558
7559
7560	#ifdef IEM_WITH_SETJMP
7561	/**
7562	* Stores a data dqword, longjmp on error.
7563	*
7564	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7565	* @param iSegReg The index of the segment register to use for
7566	* this access. The base and limits are checked.
7567	* @param GCPtrMem The address of the guest memory.
7568	* @param u128Value The value to store.
7569	*/
7570	void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7571	{
7572	/* The lazy approach for now... */
7573	PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem,
7574	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7575	pu128Dst->au64[0] = u128Value.au64[0];
7576	pu128Dst->au64[1] = u128Value.au64[1];
7577	iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7578	}
7579	#endif
7580
7581
7582	/**
7583	* Stores a data dqword, SSE aligned.
7584	*
7585	* @returns Strict VBox status code.
7586	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7587	* @param iSegReg The index of the segment register to use for
7588	* this access. The base and limits are checked.
7589	* @param GCPtrMem The address of the guest memory.
7590	* @param u128Value The value to store.
7591	*/
7592	VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7593	{
7594	/* The lazy approach for now... */
7595	PRTUINT128U pu128Dst;
7596	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Dst, sizeof(pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
7597	(sizeof(*pu128Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7598	if (rc == VINF_SUCCESS)
7599	{
7600	pu128Dst->au64[0] = u128Value.au64[0];
7601	pu128Dst->au64[1] = u128Value.au64[1];
7602	rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7603	}
7604	return rc;
7605	}
7606
7607
7608	#ifdef IEM_WITH_SETJMP
7609	/**
7610	* Stores a data dqword, SSE aligned.
7611	*
7612	* @returns Strict VBox status code.
7613	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7614	* @param iSegReg The index of the segment register to use for
7615	* this access. The base and limits are checked.
7616	* @param GCPtrMem The address of the guest memory.
7617	* @param u128Value The value to store.
7618	*/
7619	void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7620	{
7621	/* The lazy approach for now... */
7622	PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
7623	(sizeof(*pu128Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7624	pu128Dst->au64[0] = u128Value.au64[0];
7625	pu128Dst->au64[1] = u128Value.au64[1];
7626	iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7627	}
7628	#endif
7629
7630
7631	/**
7632	* Stores a data dqword.
7633	*
7634	* @returns Strict VBox status code.
7635	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7636	* @param iSegReg The index of the segment register to use for
7637	* this access. The base and limits are checked.
7638	* @param GCPtrMem The address of the guest memory.
7639	* @param pu256Value Pointer to the value to store.
7640	*/
7641	VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7642	{
7643	/* The lazy approach for now... */
7644	PRTUINT256U pu256Dst;
7645	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Dst, sizeof(pu256Dst), iSegReg, GCPtrMem,
7646	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7647	if (rc == VINF_SUCCESS)
7648	{
7649	pu256Dst->au64[0] = pu256Value->au64[0];
7650	pu256Dst->au64[1] = pu256Value->au64[1];
7651	pu256Dst->au64[2] = pu256Value->au64[2];
7652	pu256Dst->au64[3] = pu256Value->au64[3];
7653	rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7654	}
7655	return rc;
7656	}
7657
7658
7659	#ifdef IEM_WITH_SETJMP
7660	/**
7661	* Stores a data dqword, longjmp on error.
7662	*
7663	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7664	* @param iSegReg The index of the segment register to use for
7665	* this access. The base and limits are checked.
7666	* @param GCPtrMem The address of the guest memory.
7667	* @param pu256Value Pointer to the value to store.
7668	*/
7669	void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7670	{
7671	/* The lazy approach for now... */
7672	PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7673	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7674	pu256Dst->au64[0] = pu256Value->au64[0];
7675	pu256Dst->au64[1] = pu256Value->au64[1];
7676	pu256Dst->au64[2] = pu256Value->au64[2];
7677	pu256Dst->au64[3] = pu256Value->au64[3];
7678	iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7679	}
7680	#endif
7681
7682
7683	/**
7684	* Stores a data dqword, AVX \#GP(0) aligned.
7685	*
7686	* @returns Strict VBox status code.
7687	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7688	* @param iSegReg The index of the segment register to use for
7689	* this access. The base and limits are checked.
7690	* @param GCPtrMem The address of the guest memory.
7691	* @param pu256Value Pointer to the value to store.
7692	*/
7693	VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7694	{
7695	/* The lazy approach for now... */
7696	PRTUINT256U pu256Dst;
7697	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Dst, sizeof(pu256Dst), iSegReg, GCPtrMem,
7698	IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP);
7699	if (rc == VINF_SUCCESS)
7700	{
7701	pu256Dst->au64[0] = pu256Value->au64[0];
7702	pu256Dst->au64[1] = pu256Value->au64[1];
7703	pu256Dst->au64[2] = pu256Value->au64[2];
7704	pu256Dst->au64[3] = pu256Value->au64[3];
7705	rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7706	}
7707	return rc;
7708	}
7709
7710
7711	#ifdef IEM_WITH_SETJMP
7712	/**
7713	* Stores a data dqword, AVX aligned.
7714	*
7715	* @returns Strict VBox status code.
7716	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7717	* @param iSegReg The index of the segment register to use for
7718	* this access. The base and limits are checked.
7719	* @param GCPtrMem The address of the guest memory.
7720	* @param pu256Value Pointer to the value to store.
7721	*/
7722	void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7723	{
7724	/* The lazy approach for now... */
7725	PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7726	IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP);
7727	pu256Dst->au64[0] = pu256Value->au64[0];
7728	pu256Dst->au64[1] = pu256Value->au64[1];
7729	pu256Dst->au64[2] = pu256Value->au64[2];
7730	pu256Dst->au64[3] = pu256Value->au64[3];
7731	iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7732	}
7733	#endif
7734
7735
7736	/**
7737	* Stores a descriptor register (sgdt, sidt).
7738	*
7739	* @returns Strict VBox status code.
7740	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7741	* @param cbLimit The limit.
7742	* @param GCPtrBase The base address.
7743	* @param iSegReg The index of the segment register to use for
7744	* this access. The base and limits are checked.
7745	* @param GCPtrMem The address of the guest memory.
7746	*/
7747	VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7748	{
7749	/*
7750	* The SIDT and SGDT instructions actually stores the data using two
7751	* independent writes (see bs3CpuBasic2_sidt_sgdt_One). The instructions
7752	* does not respond to opsize prefixes.
7753	*/
7754	VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit);
7755	if (rcStrict == VINF_SUCCESS)
7756	{
7757	if (pVCpu->iem.s.enmCpuMode == IEMMODE_16BIT)
7758	rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2,
7759	IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286
7760	? (uint32_t)GCPtrBase \| UINT32_C(0xff000000) : (uint32_t)GCPtrBase);
7761	else if (pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT)
7762	rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase);
7763	else
7764	rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase);
7765	}
7766	return rcStrict;
7767	}
7768
7769
7770	/**
7771	* Pushes a word onto the stack.
7772	*
7773	* @returns Strict VBox status code.
7774	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7775	* @param u16Value The value to push.
7776	*/
7777	VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value) RT_NOEXCEPT
7778	{
7779	/* Increment the stack pointer. */
7780	uint64_t uNewRsp;
7781	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 2, &uNewRsp);
7782
7783	/* Write the word the lazy way. */
7784	uint16_t *pu16Dst;
7785	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop,
7786	IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1);
7787	if (rc == VINF_SUCCESS)
7788	{
7789	*pu16Dst = u16Value;
7790	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
7791	}
7792
7793	/* Commit the new RSP value unless we an access handler made trouble. */
7794	if (rc == VINF_SUCCESS)
7795	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7796
7797	return rc;
7798	}
7799
7800
7801	/**
7802	* Pushes a dword onto the stack.
7803	*
7804	* @returns Strict VBox status code.
7805	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7806	* @param u32Value The value to push.
7807	*/
7808	VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT
7809	{
7810	/* Increment the stack pointer. */
7811	uint64_t uNewRsp;
7812	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
7813
7814	/* Write the dword the lazy way. */
7815	uint32_t *pu32Dst;
7816	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop,
7817	IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1);
7818	if (rc == VINF_SUCCESS)
7819	{
7820	*pu32Dst = u32Value;
7821	rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
7822	}
7823
7824	/* Commit the new RSP value unless we an access handler made trouble. */
7825	if (rc == VINF_SUCCESS)
7826	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7827
7828	return rc;
7829	}
7830
7831
7832	/**
7833	* Pushes a dword segment register value onto the stack.
7834	*
7835	* @returns Strict VBox status code.
7836	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7837	* @param u32Value The value to push.
7838	*/
7839	VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT
7840	{
7841	/* Increment the stack pointer. */
7842	uint64_t uNewRsp;
7843	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
7844
7845	/* The intel docs talks about zero extending the selector register
7846	value. My actual intel CPU here might be zero extending the value
7847	but it still only writes the lower word... */
7848	/** @todo Test this on new HW and on AMD and in 64-bit mode. Also test what
7849	* happens when crossing an electric page boundrary, is the high word checked
7850	* for write accessibility or not? Probably it is. What about segment limits?
7851	* It appears this behavior is also shared with trap error codes.
7852	*
7853	* Docs indicate the behavior changed maybe in Pentium or Pentium Pro. Check
7854	* ancient hardware when it actually did change. */
7855	uint16_t *pu16Dst;
7856	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop,
7857	IEM_ACCESS_STACK_RW, sizeof(pu16Dst) - 1); /* @todo 2 or 4 alignment check for PUSH SS? */
7858	if (rc == VINF_SUCCESS)
7859	{
7860	*pu16Dst = (uint16_t)u32Value;
7861	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_RW);
7862	}
7863
7864	/* Commit the new RSP value unless we an access handler made trouble. */
7865	if (rc == VINF_SUCCESS)
7866	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7867
7868	return rc;
7869	}
7870
7871
7872	/**
7873	* Pushes a qword onto the stack.
7874	*
7875	* @returns Strict VBox status code.
7876	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7877	* @param u64Value The value to push.
7878	*/
7879	VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value) RT_NOEXCEPT
7880	{
7881	/* Increment the stack pointer. */
7882	uint64_t uNewRsp;
7883	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 8, &uNewRsp);
7884
7885	/* Write the word the lazy way. */
7886	uint64_t *pu64Dst;
7887	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop,
7888	IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1);
7889	if (rc == VINF_SUCCESS)
7890	{
7891	*pu64Dst = u64Value;
7892	rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
7893	}
7894
7895	/* Commit the new RSP value unless we an access handler made trouble. */
7896	if (rc == VINF_SUCCESS)
7897	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7898
7899	return rc;
7900	}
7901
7902
7903	/**
7904	* Pops a word from the stack.
7905	*
7906	* @returns Strict VBox status code.
7907	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7908	* @param pu16Value Where to store the popped value.
7909	*/
7910	VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value) RT_NOEXCEPT
7911	{
7912	/* Increment the stack pointer. */
7913	uint64_t uNewRsp;
7914	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 2, &uNewRsp);
7915
7916	/* Write the word the lazy way. */
7917	uint16_t const *pu16Src;
7918	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop,
7919	IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1);
7920	if (rc == VINF_SUCCESS)
7921	{
7922	pu16Value = pu16Src;
7923	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
7924
7925	/* Commit the new RSP value. */
7926	if (rc == VINF_SUCCESS)
7927	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7928	}
7929
7930	return rc;
7931	}
7932
7933
7934	/**
7935	* Pops a dword from the stack.
7936	*
7937	* @returns Strict VBox status code.
7938	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7939	* @param pu32Value Where to store the popped value.
7940	*/
7941	VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value) RT_NOEXCEPT
7942	{
7943	/* Increment the stack pointer. */
7944	uint64_t uNewRsp;
7945	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 4, &uNewRsp);
7946
7947	/* Write the word the lazy way. */
7948	uint32_t const *pu32Src;
7949	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop,
7950	IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1);
7951	if (rc == VINF_SUCCESS)
7952	{
7953	pu32Value = pu32Src;
7954	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
7955
7956	/* Commit the new RSP value. */
7957	if (rc == VINF_SUCCESS)
7958	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7959	}
7960
7961	return rc;
7962	}
7963
7964
7965	/**
7966	* Pops a qword from the stack.
7967	*
7968	* @returns Strict VBox status code.
7969	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7970	* @param pu64Value Where to store the popped value.
7971	*/
7972	VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value) RT_NOEXCEPT
7973	{
7974	/* Increment the stack pointer. */
7975	uint64_t uNewRsp;
7976	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 8, &uNewRsp);
7977
7978	/* Write the word the lazy way. */
7979	uint64_t const *pu64Src;
7980	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop,
7981	IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1);
7982	if (rc == VINF_SUCCESS)
7983	{
7984	pu64Value = pu64Src;
7985	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
7986
7987	/* Commit the new RSP value. */
7988	if (rc == VINF_SUCCESS)
7989	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7990	}
7991
7992	return rc;
7993	}
7994
7995
7996	/**
7997	* Pushes a word onto the stack, using a temporary stack pointer.
7998	*
7999	* @returns Strict VBox status code.
8000	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8001	* @param u16Value The value to push.
8002	* @param pTmpRsp Pointer to the temporary stack pointer.
8003	*/
8004	VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8005	{
8006	/* Increment the stack pointer. */
8007	RTUINT64U NewRsp = *pTmpRsp;
8008	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 2);
8009
8010	/* Write the word the lazy way. */
8011	uint16_t *pu16Dst;
8012	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop,
8013	IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1);
8014	if (rc == VINF_SUCCESS)
8015	{
8016	*pu16Dst = u16Value;
8017	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
8018	}
8019
8020	/* Commit the new RSP value unless we an access handler made trouble. */
8021	if (rc == VINF_SUCCESS)
8022	*pTmpRsp = NewRsp;
8023
8024	return rc;
8025	}
8026
8027
8028	/**
8029	* Pushes a dword onto the stack, using a temporary stack pointer.
8030	*
8031	* @returns Strict VBox status code.
8032	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8033	* @param u32Value The value to push.
8034	* @param pTmpRsp Pointer to the temporary stack pointer.
8035	*/
8036	VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8037	{
8038	/* Increment the stack pointer. */
8039	RTUINT64U NewRsp = *pTmpRsp;
8040	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 4);
8041
8042	/* Write the word the lazy way. */
8043	uint32_t *pu32Dst;
8044	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop,
8045	IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1);
8046	if (rc == VINF_SUCCESS)
8047	{
8048	*pu32Dst = u32Value;
8049	rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
8050	}
8051
8052	/* Commit the new RSP value unless we an access handler made trouble. */
8053	if (rc == VINF_SUCCESS)
8054	*pTmpRsp = NewRsp;
8055
8056	return rc;
8057	}
8058
8059
8060	/**
8061	* Pushes a dword onto the stack, using a temporary stack pointer.
8062	*
8063	* @returns Strict VBox status code.
8064	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8065	* @param u64Value The value to push.
8066	* @param pTmpRsp Pointer to the temporary stack pointer.
8067	*/
8068	VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8069	{
8070	/* Increment the stack pointer. */
8071	RTUINT64U NewRsp = *pTmpRsp;
8072	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 8);
8073
8074	/* Write the word the lazy way. */
8075	uint64_t *pu64Dst;
8076	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop,
8077	IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1);
8078	if (rc == VINF_SUCCESS)
8079	{
8080	*pu64Dst = u64Value;
8081	rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
8082	}
8083
8084	/* Commit the new RSP value unless we an access handler made trouble. */
8085	if (rc == VINF_SUCCESS)
8086	*pTmpRsp = NewRsp;
8087
8088	return rc;
8089	}
8090
8091
8092	/**
8093	* Pops a word from the stack, using a temporary stack pointer.
8094	*
8095	* @returns Strict VBox status code.
8096	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8097	* @param pu16Value Where to store the popped value.
8098	* @param pTmpRsp Pointer to the temporary stack pointer.
8099	*/
8100	VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8101	{
8102	/* Increment the stack pointer. */
8103	RTUINT64U NewRsp = *pTmpRsp;
8104	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 2);
8105
8106	/* Write the word the lazy way. */
8107	uint16_t const *pu16Src;
8108	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop,
8109	IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1);
8110	if (rc == VINF_SUCCESS)
8111	{
8112	pu16Value = pu16Src;
8113	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
8114
8115	/* Commit the new RSP value. */
8116	if (rc == VINF_SUCCESS)
8117	*pTmpRsp = NewRsp;
8118	}
8119
8120	return rc;
8121	}
8122
8123
8124	/**
8125	* Pops a dword from the stack, using a temporary stack pointer.
8126	*
8127	* @returns Strict VBox status code.
8128	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8129	* @param pu32Value Where to store the popped value.
8130	* @param pTmpRsp Pointer to the temporary stack pointer.
8131	*/
8132	VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8133	{
8134	/* Increment the stack pointer. */
8135	RTUINT64U NewRsp = *pTmpRsp;
8136	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 4);
8137
8138	/* Write the word the lazy way. */
8139	uint32_t const *pu32Src;
8140	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop,
8141	IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1);
8142	if (rc == VINF_SUCCESS)
8143	{
8144	pu32Value = pu32Src;
8145	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
8146
8147	/* Commit the new RSP value. */
8148	if (rc == VINF_SUCCESS)
8149	*pTmpRsp = NewRsp;
8150	}
8151
8152	return rc;
8153	}
8154
8155
8156	/**
8157	* Pops a qword from the stack, using a temporary stack pointer.
8158	*
8159	* @returns Strict VBox status code.
8160	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8161	* @param pu64Value Where to store the popped value.
8162	* @param pTmpRsp Pointer to the temporary stack pointer.
8163	*/
8164	VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8165	{
8166	/* Increment the stack pointer. */
8167	RTUINT64U NewRsp = *pTmpRsp;
8168	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
8169
8170	/* Write the word the lazy way. */
8171	uint64_t const *pu64Src;
8172	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop,
8173	IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1);
8174	if (rcStrict == VINF_SUCCESS)
8175	{
8176	pu64Value = pu64Src;
8177	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
8178
8179	/* Commit the new RSP value. */
8180	if (rcStrict == VINF_SUCCESS)
8181	*pTmpRsp = NewRsp;
8182	}
8183
8184	return rcStrict;
8185	}
8186
8187
8188	/**
8189	* Begin a special stack push (used by interrupt, exceptions and such).
8190	*
8191	* This will raise \#SS or \#PF if appropriate.
8192	*
8193	* @returns Strict VBox status code.
8194	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8195	* @param cbMem The number of bytes to push onto the stack.
8196	* @param cbAlign The alignment mask (7, 3, 1).
8197	* @param ppvMem Where to return the pointer to the stack memory.
8198	* As with the other memory functions this could be
8199	* direct access or bounce buffered access, so
8200	* don't commit register until the commit call
8201	* succeeds.
8202	* @param puNewRsp Where to return the new RSP value. This must be
8203	* passed unchanged to
8204	* iemMemStackPushCommitSpecial().
8205	*/
8206	VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
8207	void *ppvMem, uint64_t puNewRsp) RT_NOEXCEPT
8208	{
8209	Assert(cbMem < UINT8_MAX);
8210	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp);
8211	return iemMemMap(pVCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop,
8212	IEM_ACCESS_STACK_W, cbAlign);
8213	}
8214
8215
8216	/**
8217	* Commits a special stack push (started by iemMemStackPushBeginSpecial).
8218	*
8219	* This will update the rSP.
8220	*
8221	* @returns Strict VBox status code.
8222	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8223	* @param pvMem The pointer returned by
8224	* iemMemStackPushBeginSpecial().
8225	* @param uNewRsp The new RSP value returned by
8226	* iemMemStackPushBeginSpecial().
8227	*/
8228	VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp) RT_NOEXCEPT
8229	{
8230	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem, IEM_ACCESS_STACK_W);
8231	if (rcStrict == VINF_SUCCESS)
8232	pVCpu->cpum.GstCtx.rsp = uNewRsp;
8233	return rcStrict;
8234	}
8235
8236
8237	/**
8238	* Begin a special stack pop (used by iret, retf and such).
8239	*
8240	* This will raise \#SS or \#PF if appropriate.
8241	*
8242	* @returns Strict VBox status code.
8243	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8244	* @param cbMem The number of bytes to pop from the stack.
8245	* @param cbAlign The alignment mask (7, 3, 1).
8246	* @param ppvMem Where to return the pointer to the stack memory.
8247	* @param puNewRsp Where to return the new RSP value. This must be
8248	* assigned to CPUMCTX::rsp manually some time
8249	* after iemMemStackPopDoneSpecial() has been
8250	* called.
8251	*/
8252	VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
8253	void const *ppvMem, uint64_t puNewRsp) RT_NOEXCEPT
8254	{
8255	Assert(cbMem < UINT8_MAX);
8256	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp);
8257	return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R, cbAlign);
8258	}
8259
8260
8261	/**
8262	* Continue a special stack pop (used by iret and retf), for the purpose of
8263	* retrieving a new stack pointer.
8264	*
8265	* This will raise \#SS or \#PF if appropriate.
8266	*
8267	* @returns Strict VBox status code.
8268	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8269	* @param off Offset from the top of the stack. This is zero
8270	* except in the retf case.
8271	* @param cbMem The number of bytes to pop from the stack.
8272	* @param ppvMem Where to return the pointer to the stack memory.
8273	* @param uCurNewRsp The current uncommitted RSP value. (No need to
8274	* return this because all use of this function is
8275	* to retrieve a new value and anything we return
8276	* here would be discarded.)
8277	*/
8278	VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t off, size_t cbMem,
8279	void const **ppvMem, uint64_t uCurNewRsp) RT_NOEXCEPT
8280	{
8281	Assert(cbMem < UINT8_MAX);
8282
8283	/* The essense of iemRegGetRspForPopEx and friends: / /* @todo put this into a inlined function? */
8284	RTGCPTR GCPtrTop;
8285	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
8286	GCPtrTop = uCurNewRsp;
8287	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
8288	GCPtrTop = (uint32_t)uCurNewRsp;
8289	else
8290	GCPtrTop = (uint16_t)uCurNewRsp;
8291
8292	return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop + off, IEM_ACCESS_STACK_R,
8293	0 /* checked in iemMemStackPopBeginSpecial */);
8294	}
8295
8296
8297	/**
8298	* Done with a special stack pop (started by iemMemStackPopBeginSpecial or
8299	* iemMemStackPopContinueSpecial).
8300	*
8301	* The caller will manually commit the rSP.
8302	*
8303	* @returns Strict VBox status code.
8304	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8305	* @param pvMem The pointer returned by
8306	* iemMemStackPopBeginSpecial() or
8307	* iemMemStackPopContinueSpecial().
8308	*/
8309	VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem) RT_NOEXCEPT
8310	{
8311	return iemMemCommitAndUnmap(pVCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
8312	}
8313
8314
8315	/**
8316	* Fetches a system table byte.
8317	*
8318	* @returns Strict VBox status code.
8319	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8320	* @param pbDst Where to return the byte.
8321	* @param iSegReg The index of the segment register to use for
8322	* this access. The base and limits are checked.
8323	* @param GCPtrMem The address of the guest memory.
8324	*/
8325	VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8326	{
8327	/* The lazy approach for now... */
8328	uint8_t const *pbSrc;
8329	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pbSrc, sizeof(pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8330	if (rc == VINF_SUCCESS)
8331	{
8332	pbDst = pbSrc;
8333	rc = iemMemCommitAndUnmap(pVCpu, (void *)pbSrc, IEM_ACCESS_SYS_R);
8334	}
8335	return rc;
8336	}
8337
8338
8339	/**
8340	* Fetches a system table word.
8341	*
8342	* @returns Strict VBox status code.
8343	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8344	* @param pu16Dst Where to return the word.
8345	* @param iSegReg The index of the segment register to use for
8346	* this access. The base and limits are checked.
8347	* @param GCPtrMem The address of the guest memory.
8348	*/
8349	VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8350	{
8351	/* The lazy approach for now... */
8352	uint16_t const *pu16Src;
8353	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8354	if (rc == VINF_SUCCESS)
8355	{
8356	pu16Dst = pu16Src;
8357	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_SYS_R);
8358	}
8359	return rc;
8360	}
8361
8362
8363	/**
8364	* Fetches a system table dword.
8365	*
8366	* @returns Strict VBox status code.
8367	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8368	* @param pu32Dst Where to return the dword.
8369	* @param iSegReg The index of the segment register to use for
8370	* this access. The base and limits are checked.
8371	* @param GCPtrMem The address of the guest memory.
8372	*/
8373	VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8374	{
8375	/* The lazy approach for now... */
8376	uint32_t const *pu32Src;
8377	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8378	if (rc == VINF_SUCCESS)
8379	{
8380	pu32Dst = pu32Src;
8381	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
8382	}
8383	return rc;
8384	}
8385
8386
8387	/**
8388	* Fetches a system table qword.
8389	*
8390	* @returns Strict VBox status code.
8391	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8392	* @param pu64Dst Where to return the qword.
8393	* @param iSegReg The index of the segment register to use for
8394	* this access. The base and limits are checked.
8395	* @param GCPtrMem The address of the guest memory.
8396	*/
8397	VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8398	{
8399	/* The lazy approach for now... */
8400	uint64_t const *pu64Src;
8401	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8402	if (rc == VINF_SUCCESS)
8403	{
8404	pu64Dst = pu64Src;
8405	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
8406	}
8407	return rc;
8408	}
8409
8410
8411	/**
8412	* Fetches a descriptor table entry with caller specified error code.
8413	*
8414	* @returns Strict VBox status code.
8415	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8416	* @param pDesc Where to return the descriptor table entry.
8417	* @param uSel The selector which table entry to fetch.
8418	* @param uXcpt The exception to raise on table lookup error.
8419	* @param uErrorCode The error code associated with the exception.
8420	*/
8421	static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
8422	uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT
8423	{
8424	AssertPtr(pDesc);
8425	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
8426
8427	/** @todo did the 286 require all 8 bytes to be accessible? */
8428	/*
8429	* Get the selector table base and check bounds.
8430	*/
8431	RTGCPTR GCPtrBase;
8432	if (uSel & X86_SEL_LDT)
8433	{
8434	if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
8435	\|\| (uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
8436	{
8437	Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
8438	uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel));
8439	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
8440	uErrorCode, 0);
8441	}
8442
8443	Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);
8444	GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
8445	}
8446	else
8447	{
8448	if ((uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
8449	{
8450	Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt));
8451	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
8452	uErrorCode, 0);
8453	}
8454	GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
8455	}
8456
8457	/*
8458	* Read the legacy descriptor and maybe the long mode extensions if
8459	* required.
8460	*/
8461	VBOXSTRICTRC rcStrict;
8462	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286)
8463	rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
8464	else
8465	{
8466	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0);
8467	if (rcStrict == VINF_SUCCESS)
8468	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2);
8469	if (rcStrict == VINF_SUCCESS)
8470	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4);
8471	if (rcStrict == VINF_SUCCESS)
8472	pDesc->Legacy.au16[3] = 0;
8473	else
8474	return rcStrict;
8475	}
8476
8477	if (rcStrict == VINF_SUCCESS)
8478	{
8479	if ( !IEM_IS_LONG_MODE(pVCpu)
8480	\|\| pDesc->Legacy.Gen.u1DescType)
8481	pDesc->Long.au64[1] = 0;
8482	else if ((uint32_t)(uSel \| X86_SEL_RPL_LDT) + 8 <= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt))
8483	rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel \| X86_SEL_RPL_LDT) + 1);
8484	else
8485	{
8486	Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
8487	/** @todo is this the right exception? */
8488	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
8489	}
8490	}
8491	return rcStrict;
8492	}
8493
8494
8495	/**
8496	* Fetches a descriptor table entry.
8497	*
8498	* @returns Strict VBox status code.
8499	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8500	* @param pDesc Where to return the descriptor table entry.
8501	* @param uSel The selector which table entry to fetch.
8502	* @param uXcpt The exception to raise on table lookup error.
8503	*/
8504	VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT
8505	{
8506	return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
8507	}
8508
8509
8510	/**
8511	* Marks the selector descriptor as accessed (only non-system descriptors).
8512	*
8513	* This function ASSUMES that iemMemFetchSelDesc has be called previously and
8514	* will therefore skip the limit checks.
8515	*
8516	* @returns Strict VBox status code.
8517	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8518	* @param uSel The selector.
8519	*/
8520	VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
8521	{
8522	/*
8523	* Get the selector table base and calculate the entry address.
8524	*/
8525	RTGCPTR GCPtr = uSel & X86_SEL_LDT
8526	? pVCpu->cpum.GstCtx.ldtr.u64Base
8527	: pVCpu->cpum.GstCtx.gdtr.pGdt;
8528	GCPtr += uSel & X86_SEL_MASK;
8529
8530	/*
8531	* ASMAtomicBitSet will assert if the address is misaligned, so do some
8532	* ugly stuff to avoid this. This will make sure it's an atomic access
8533	* as well more or less remove any question about 8-bit or 32-bit accesss.
8534	*/
8535	VBOXSTRICTRC rcStrict;
8536	uint32_t volatile *pu32;
8537	if ((GCPtr & 3) == 0)
8538	{
8539	/* The normal case, map the 32-bit bits around the accessed bit (40). */
8540	GCPtr += 2 + 2;
8541	rcStrict = iemMemMap(pVCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
8542	if (rcStrict != VINF_SUCCESS)
8543	return rcStrict;
8544	ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
8545	}
8546	else
8547	{
8548	/* The misaligned GDT/LDT case, map the whole thing. */
8549	rcStrict = iemMemMap(pVCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
8550	if (rcStrict != VINF_SUCCESS)
8551	return rcStrict;
8552	switch ((uintptr_t)pu32 & 3)
8553	{
8554	case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
8555	case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
8556	case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
8557	case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
8558	}
8559	}
8560
8561	return iemMemCommitAndUnmap(pVCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
8562	}
8563
8564	/** @} */
8565
8566	/** @name Opcode Helpers.
8567	* @{
8568	*/
8569
8570	/**
8571	* Calculates the effective address of a ModR/M memory operand.
8572	*
8573	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8574	*
8575	* @return Strict VBox status code.
8576	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8577	* @param bRm The ModRM byte.
8578	* @param cbImm The size of any immediate following the
8579	* effective address opcode bytes. Important for
8580	* RIP relative addressing.
8581	* @param pGCPtrEff Where to return the effective address.
8582	*/
8583	VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff) RT_NOEXCEPT
8584	{
8585	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
8586	# define SET_SS_DEF() \
8587	do \
8588	{ \
8589	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8590	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8591	} while (0)
8592
8593	if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
8594	{
8595	/** @todo Check the effective address size crap! */
8596	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8597	{
8598	uint16_t u16EffAddr;
8599
8600	/* Handle the disp16 form with no registers first. */
8601	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
8602	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8603	else
8604	{
8605	/* Get the displacment. */
8606	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8607	{
8608	case 0: u16EffAddr = 0; break;
8609	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8610	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8611	default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
8612	}
8613
8614	/* Add the base and index registers to the disp. */
8615	switch (bRm & X86_MODRM_RM_MASK)
8616	{
8617	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8618	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8619	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8620	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8621	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8622	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8623	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8624	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8625	}
8626	}
8627
8628	*pGCPtrEff = u16EffAddr;
8629	}
8630	else
8631	{
8632	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8633	uint32_t u32EffAddr;
8634
8635	/* Handle the disp32 form with no registers first. */
8636	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8637	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8638	else
8639	{
8640	/* Get the register (or SIB) value. */
8641	switch ((bRm & X86_MODRM_RM_MASK))
8642	{
8643	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8644	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8645	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8646	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8647	case 4: /* SIB */
8648	{
8649	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8650
8651	/* Get the index and scale it. */
8652	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8653	{
8654	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8655	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8656	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8657	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8658	case 4: u32EffAddr = 0; /none / break;
8659	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8660	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8661	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8662	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8663	}
8664	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8665
8666	/* add base */
8667	switch (bSib & X86_SIB_BASE_MASK)
8668	{
8669	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8670	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8671	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8672	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8673	case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
8674	case 5:
8675	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8676	{
8677	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8678	SET_SS_DEF();
8679	}
8680	else
8681	{
8682	uint32_t u32Disp;
8683	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8684	u32EffAddr += u32Disp;
8685	}
8686	break;
8687	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
8688	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
8689	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8690	}
8691	break;
8692	}
8693	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
8694	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8695	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8696	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8697	}
8698
8699	/* Get and add the displacement. */
8700	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8701	{
8702	case 0:
8703	break;
8704	case 1:
8705	{
8706	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8707	u32EffAddr += i8Disp;
8708	break;
8709	}
8710	case 2:
8711	{
8712	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8713	u32EffAddr += u32Disp;
8714	break;
8715	}
8716	default:
8717	AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
8718	}
8719
8720	}
8721	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
8722	*pGCPtrEff = u32EffAddr;
8723	else
8724	{
8725	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
8726	*pGCPtrEff = u32EffAddr & UINT16_MAX;
8727	}
8728	}
8729	}
8730	else
8731	{
8732	uint64_t u64EffAddr;
8733
8734	/* Handle the rip+disp32 form with no registers first. */
8735	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8736	{
8737	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
8738	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
8739	}
8740	else
8741	{
8742	/* Get the register (or SIB) value. */
8743	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
8744	{
8745	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8746	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8747	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8748	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8749	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
8750	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8751	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8752	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8753	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8754	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8755	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8756	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8757	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8758	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8759	/* SIB */
8760	case 4:
8761	case 12:
8762	{
8763	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8764
8765	/* Get the index and scale it. */
8766	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
8767	{
8768	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8769	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8770	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8771	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8772	case 4: u64EffAddr = 0; /none / break;
8773	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
8774	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8775	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8776	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8777	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8778	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8779	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8780	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
8781	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8782	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8783	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8784	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8785	}
8786	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8787
8788	/* add base */
8789	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
8790	{
8791	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
8792	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
8793	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
8794	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
8795	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
8796	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
8797	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
8798	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
8799	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
8800	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
8801	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
8802	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
8803	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
8804	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
8805	/* complicated encodings */
8806	case 5:
8807	case 13:
8808	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8809	{
8810	if (!pVCpu->iem.s.uRexB)
8811	{
8812	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
8813	SET_SS_DEF();
8814	}
8815	else
8816	u64EffAddr += pVCpu->cpum.GstCtx.r13;
8817	}
8818	else
8819	{
8820	uint32_t u32Disp;
8821	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8822	u64EffAddr += (int32_t)u32Disp;
8823	}
8824	break;
8825	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8826	}
8827	break;
8828	}
8829	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8830	}
8831
8832	/* Get and add the displacement. */
8833	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8834	{
8835	case 0:
8836	break;
8837	case 1:
8838	{
8839	int8_t i8Disp;
8840	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8841	u64EffAddr += i8Disp;
8842	break;
8843	}
8844	case 2:
8845	{
8846	uint32_t u32Disp;
8847	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8848	u64EffAddr += (int32_t)u32Disp;
8849	break;
8850	}
8851	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
8852	}
8853
8854	}
8855
8856	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
8857	*pGCPtrEff = u64EffAddr;
8858	else
8859	{
8860	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8861	*pGCPtrEff = u64EffAddr & UINT32_MAX;
8862	}
8863	}
8864
8865	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
8866	return VINF_SUCCESS;
8867	}
8868
8869
8870	/**
8871	* Calculates the effective address of a ModR/M memory operand.
8872	*
8873	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8874	*
8875	* @return Strict VBox status code.
8876	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8877	* @param bRm The ModRM byte.
8878	* @param cbImm The size of any immediate following the
8879	* effective address opcode bytes. Important for
8880	* RIP relative addressing.
8881	* @param pGCPtrEff Where to return the effective address.
8882	* @param offRsp RSP displacement.
8883	*/
8884	VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff, int8_t offRsp) RT_NOEXCEPT
8885	{
8886	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
8887	# define SET_SS_DEF() \
8888	do \
8889	{ \
8890	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8891	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8892	} while (0)
8893
8894	if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
8895	{
8896	/** @todo Check the effective address size crap! */
8897	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8898	{
8899	uint16_t u16EffAddr;
8900
8901	/* Handle the disp16 form with no registers first. */
8902	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
8903	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8904	else
8905	{
8906	/* Get the displacment. */
8907	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8908	{
8909	case 0: u16EffAddr = 0; break;
8910	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8911	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8912	default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
8913	}
8914
8915	/* Add the base and index registers to the disp. */
8916	switch (bRm & X86_MODRM_RM_MASK)
8917	{
8918	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8919	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8920	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8921	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8922	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8923	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8924	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8925	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8926	}
8927	}
8928
8929	*pGCPtrEff = u16EffAddr;
8930	}
8931	else
8932	{
8933	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8934	uint32_t u32EffAddr;
8935
8936	/* Handle the disp32 form with no registers first. */
8937	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8938	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8939	else
8940	{
8941	/* Get the register (or SIB) value. */
8942	switch ((bRm & X86_MODRM_RM_MASK))
8943	{
8944	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8945	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8946	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8947	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8948	case 4: /* SIB */
8949	{
8950	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8951
8952	/* Get the index and scale it. */
8953	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8954	{
8955	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8956	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8957	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8958	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8959	case 4: u32EffAddr = 0; /none / break;
8960	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8961	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8962	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8963	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8964	}
8965	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8966
8967	/* add base */
8968	switch (bSib & X86_SIB_BASE_MASK)
8969	{
8970	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8971	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8972	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8973	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8974	case 4:
8975	u32EffAddr += pVCpu->cpum.GstCtx.esp + offRsp;
8976	SET_SS_DEF();
8977	break;
8978	case 5:
8979	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8980	{
8981	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8982	SET_SS_DEF();
8983	}
8984	else
8985	{
8986	uint32_t u32Disp;
8987	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8988	u32EffAddr += u32Disp;
8989	}
8990	break;
8991	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
8992	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
8993	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8994	}
8995	break;
8996	}
8997	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
8998	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8999	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9000	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9001	}
9002
9003	/* Get and add the displacement. */
9004	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9005	{
9006	case 0:
9007	break;
9008	case 1:
9009	{
9010	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9011	u32EffAddr += i8Disp;
9012	break;
9013	}
9014	case 2:
9015	{
9016	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9017	u32EffAddr += u32Disp;
9018	break;
9019	}
9020	default:
9021	AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
9022	}
9023
9024	}
9025	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
9026	*pGCPtrEff = u32EffAddr;
9027	else
9028	{
9029	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
9030	*pGCPtrEff = u32EffAddr & UINT16_MAX;
9031	}
9032	}
9033	}
9034	else
9035	{
9036	uint64_t u64EffAddr;
9037
9038	/* Handle the rip+disp32 form with no registers first. */
9039	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9040	{
9041	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
9042	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
9043	}
9044	else
9045	{
9046	/* Get the register (or SIB) value. */
9047	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
9048	{
9049	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9050	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9051	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9052	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9053	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
9054	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9055	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9056	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9057	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9058	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9059	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9060	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9061	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9062	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9063	/* SIB */
9064	case 4:
9065	case 12:
9066	{
9067	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9068
9069	/* Get the index and scale it. */
9070	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
9071	{
9072	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9073	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9074	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9075	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9076	case 4: u64EffAddr = 0; /none / break;
9077	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
9078	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9079	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9080	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9081	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9082	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9083	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9084	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
9085	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9086	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9087	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9088	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9089	}
9090	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9091
9092	/* add base */
9093	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
9094	{
9095	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
9096	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
9097	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
9098	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
9099	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + offRsp; SET_SS_DEF(); break;
9100	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
9101	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
9102	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
9103	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
9104	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
9105	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
9106	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
9107	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
9108	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
9109	/* complicated encodings */
9110	case 5:
9111	case 13:
9112	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9113	{
9114	if (!pVCpu->iem.s.uRexB)
9115	{
9116	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
9117	SET_SS_DEF();
9118	}
9119	else
9120	u64EffAddr += pVCpu->cpum.GstCtx.r13;
9121	}
9122	else
9123	{
9124	uint32_t u32Disp;
9125	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9126	u64EffAddr += (int32_t)u32Disp;
9127	}
9128	break;
9129	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9130	}
9131	break;
9132	}
9133	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9134	}
9135
9136	/* Get and add the displacement. */
9137	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9138	{
9139	case 0:
9140	break;
9141	case 1:
9142	{
9143	int8_t i8Disp;
9144	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9145	u64EffAddr += i8Disp;
9146	break;
9147	}
9148	case 2:
9149	{
9150	uint32_t u32Disp;
9151	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9152	u64EffAddr += (int32_t)u32Disp;
9153	break;
9154	}
9155	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
9156	}
9157
9158	}
9159
9160	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
9161	*pGCPtrEff = u64EffAddr;
9162	else
9163	{
9164	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9165	*pGCPtrEff = u64EffAddr & UINT32_MAX;
9166	}
9167	}
9168
9169	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
9170	return VINF_SUCCESS;
9171	}
9172
9173
9174	#ifdef IEM_WITH_SETJMP
9175	/**
9176	* Calculates the effective address of a ModR/M memory operand.
9177	*
9178	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
9179	*
9180	* May longjmp on internal error.
9181	*
9182	* @return The effective address.
9183	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9184	* @param bRm The ModRM byte.
9185	* @param cbImm The size of any immediate following the
9186	* effective address opcode bytes. Important for
9187	* RIP relative addressing.
9188	*/
9189	RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm) RT_NOEXCEPT
9190	{
9191	Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
9192	# define SET_SS_DEF() \
9193	do \
9194	{ \
9195	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
9196	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
9197	} while (0)
9198
9199	if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
9200	{
9201	/** @todo Check the effective address size crap! */
9202	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
9203	{
9204	uint16_t u16EffAddr;
9205
9206	/* Handle the disp16 form with no registers first. */
9207	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
9208	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
9209	else
9210	{
9211	/* Get the displacment. */
9212	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9213	{
9214	case 0: u16EffAddr = 0; break;
9215	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
9216	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
9217	default: AssertFailedStmt(longjmp(pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_1)); / (caller checked for these) */
9218	}
9219
9220	/* Add the base and index registers to the disp. */
9221	switch (bRm & X86_MODRM_RM_MASK)
9222	{
9223	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
9224	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
9225	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
9226	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
9227	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
9228	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
9229	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
9230	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
9231	}
9232	}
9233
9234	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr));
9235	return u16EffAddr;
9236	}
9237
9238	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9239	uint32_t u32EffAddr;
9240
9241	/* Handle the disp32 form with no registers first. */
9242	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9243	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
9244	else
9245	{
9246	/* Get the register (or SIB) value. */
9247	switch ((bRm & X86_MODRM_RM_MASK))
9248	{
9249	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
9250	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
9251	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
9252	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
9253	case 4: /* SIB */
9254	{
9255	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9256
9257	/* Get the index and scale it. */
9258	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
9259	{
9260	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
9261	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
9262	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
9263	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
9264	case 4: u32EffAddr = 0; /none / break;
9265	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
9266	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
9267	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9268	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9269	}
9270	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9271
9272	/* add base */
9273	switch (bSib & X86_SIB_BASE_MASK)
9274	{
9275	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
9276	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
9277	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
9278	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
9279	case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
9280	case 5:
9281	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9282	{
9283	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
9284	SET_SS_DEF();
9285	}
9286	else
9287	{
9288	uint32_t u32Disp;
9289	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9290	u32EffAddr += u32Disp;
9291	}
9292	break;
9293	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
9294	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
9295	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9296	}
9297	break;
9298	}
9299	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
9300	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
9301	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9302	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9303	}
9304
9305	/* Get and add the displacement. */
9306	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9307	{
9308	case 0:
9309	break;
9310	case 1:
9311	{
9312	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9313	u32EffAddr += i8Disp;
9314	break;
9315	}
9316	case 2:
9317	{
9318	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9319	u32EffAddr += u32Disp;
9320	break;
9321	}
9322	default:
9323	AssertFailedStmt(longjmp(pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_2)); / (caller checked for these) */
9324	}
9325	}
9326
9327	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
9328	{
9329	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr));
9330	return u32EffAddr;
9331	}
9332	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
9333	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX32\n", u32EffAddr & UINT16_MAX));
9334	return u32EffAddr & UINT16_MAX;
9335	}
9336
9337	uint64_t u64EffAddr;
9338
9339	/* Handle the rip+disp32 form with no registers first. */
9340	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9341	{
9342	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
9343	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
9344	}
9345	else
9346	{
9347	/* Get the register (or SIB) value. */
9348	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
9349	{
9350	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9351	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9352	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9353	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9354	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
9355	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9356	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9357	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9358	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9359	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9360	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9361	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9362	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9363	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9364	/* SIB */
9365	case 4:
9366	case 12:
9367	{
9368	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9369
9370	/* Get the index and scale it. */
9371	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
9372	{
9373	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9374	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9375	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9376	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9377	case 4: u64EffAddr = 0; /none / break;
9378	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
9379	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9380	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9381	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9382	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9383	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9384	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9385	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
9386	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9387	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9388	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9389	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9390	}
9391	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9392
9393	/* add base */
9394	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
9395	{
9396	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
9397	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
9398	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
9399	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
9400	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
9401	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
9402	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
9403	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
9404	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
9405	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
9406	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
9407	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
9408	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
9409	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
9410	/* complicated encodings */
9411	case 5:
9412	case 13:
9413	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9414	{
9415	if (!pVCpu->iem.s.uRexB)
9416	{
9417	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
9418	SET_SS_DEF();
9419	}
9420	else
9421	u64EffAddr += pVCpu->cpum.GstCtx.r13;
9422	}
9423	else
9424	{
9425	uint32_t u32Disp;
9426	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9427	u64EffAddr += (int32_t)u32Disp;
9428	}
9429	break;
9430	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9431	}
9432	break;
9433	}
9434	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9435	}
9436
9437	/* Get and add the displacement. */
9438	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9439	{
9440	case 0:
9441	break;
9442	case 1:
9443	{
9444	int8_t i8Disp;
9445	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9446	u64EffAddr += i8Disp;
9447	break;
9448	}
9449	case 2:
9450	{
9451	uint32_t u32Disp;
9452	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9453	u64EffAddr += (int32_t)u32Disp;
9454	break;
9455	}
9456	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
9457	}
9458
9459	}
9460
9461	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
9462	{
9463	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr));
9464	return u64EffAddr;
9465	}
9466	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9467	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX));
9468	return u64EffAddr & UINT32_MAX;
9469	}
9470	#endif /* IEM_WITH_SETJMP */
9471
9472	/** @} */
9473
9474
9475	#ifdef LOG_ENABLED
9476	/**
9477	* Logs the current instruction.
9478	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9479	* @param fSameCtx Set if we have the same context information as the VMM,
9480	* clear if we may have already executed an instruction in
9481	* our debug context. When clear, we assume IEMCPU holds
9482	* valid CPU mode info.
9483	*
9484	* The @a fSameCtx parameter is now misleading and obsolete.
9485	* @param pszFunction The IEM function doing the execution.
9486	*/
9487	static void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction) RT_NOEXCEPT
9488	{
9489	# ifdef IN_RING3
9490	if (LogIs2Enabled())
9491	{
9492	char szInstr[256];
9493	uint32_t cbInstr = 0;
9494	if (fSameCtx)
9495	DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
9496	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
9497	szInstr, sizeof(szInstr), &cbInstr);
9498	else
9499	{
9500	uint32_t fFlags = 0;
9501	switch (pVCpu->iem.s.enmCpuMode)
9502	{
9503	case IEMMODE_64BIT: fFlags \|= DBGF_DISAS_FLAGS_64BIT_MODE; break;
9504	case IEMMODE_32BIT: fFlags \|= DBGF_DISAS_FLAGS_32BIT_MODE; break;
9505	case IEMMODE_16BIT:
9506	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
9507	fFlags \|= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
9508	else
9509	fFlags \|= DBGF_DISAS_FLAGS_16BIT_MODE;
9510	break;
9511	}
9512	DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,
9513	szInstr, sizeof(szInstr), &cbInstr);
9514	}
9515
9516	PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9517	Log2(("**** %s\n"
9518	" eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
9519	" eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
9520	" cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
9521	" fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
9522	" %s\n"
9523	, pszFunction,
9524	pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
9525	pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
9526	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
9527	pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
9528	pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
9529	szInstr));
9530
9531	if (LogIs3Enabled())
9532	DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
9533	}
9534	else
9535	# endif
9536	LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
9537	pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
9538	RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);
9539	}
9540	#endif /* LOG_ENABLED */
9541
9542
9543	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9544	/**
9545	* Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW,
9546	* VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW.
9547	*
9548	* @returns Modified rcStrict.
9549	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9550	* @param rcStrict The instruction execution status.
9551	*/
9552	static VBOXSTRICTRC iemHandleNestedInstructionBoundraryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
9553	{
9554	Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)));
9555	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF))
9556	{
9557	/* VMX preemption timer takes priority over NMI-window exits. */
9558	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
9559	{
9560	rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
9561	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
9562	}
9563	/*
9564	* Check remaining intercepts.
9565	*
9566	* NMI-window and Interrupt-window VM-exits.
9567	* Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs.
9568	* Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits.
9569	*
9570	* See Intel spec. 26.7.6 "NMI-Window Exiting".
9571	* See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9572	*/
9573	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW \| VMCPU_FF_VMX_INT_WINDOW)
9574	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
9575	&& !TRPMHasTrap(pVCpu))
9576	{
9577	Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
9578	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
9579	&& CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))
9580	{
9581	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
9582	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
9583	}
9584	else if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
9585	&& CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
9586	{
9587	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
9588	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
9589	}
9590	}
9591	}
9592	/* TPR-below threshold/APIC write has the highest priority. */
9593	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
9594	{
9595	rcStrict = iemVmxApicWriteEmulation(pVCpu);
9596	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9597	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
9598	}
9599	/* MTF takes priority over VMX-preemption timer. */
9600	else
9601	{
9602	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */);
9603	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9604	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
9605	}
9606	return rcStrict;
9607	}
9608	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9609
9610
9611	/**
9612	* The actual code execution bits of IEMExecOne, IEMExecOneEx, and
9613	* IEMExecOneWithPrefetchedByPC.
9614	*
9615	* Similar code is found in IEMExecLots.
9616	*
9617	* @return Strict VBox status code.
9618	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9619	* @param fExecuteInhibit If set, execute the instruction following CLI,
9620	* POP SS and MOV SS,GR.
9621	* @param pszFunction The calling function name.
9622	*/
9623	DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction)
9624	{
9625	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9626	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9627	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9628	RT_NOREF_PV(pszFunction);
9629
9630	#ifdef IEM_WITH_SETJMP
9631	VBOXSTRICTRC rcStrict;
9632	jmp_buf JmpBuf;
9633	jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
9634	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
9635	if ((rcStrict = setjmp(JmpBuf)) == 0)
9636	{
9637	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9638	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9639	}
9640	else
9641	pVCpu->iem.s.cLongJumps++;
9642	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
9643	#else
9644	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9645	VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9646	#endif
9647	if (rcStrict == VINF_SUCCESS)
9648	pVCpu->iem.s.cInstructions++;
9649	if (pVCpu->iem.s.cActiveMappings > 0)
9650	{
9651	Assert(rcStrict != VINF_SUCCESS);
9652	iemMemRollback(pVCpu);
9653	}
9654	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9655	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9656	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9657
9658	//#ifdef DEBUG
9659	// AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr \|\| rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));
9660	//#endif
9661
9662	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9663	/*
9664	* Perform any VMX nested-guest instruction boundary actions.
9665	*
9666	* If any of these causes a VM-exit, we must skip executing the next
9667	* instruction (would run into stale page tables). A VM-exit makes sure
9668	* there is no interrupt-inhibition, so that should ensure we don't go
9669	* to try execute the next instruction. Clearing fExecuteInhibit is
9670	* problematic because of the setjmp/longjmp clobbering above.
9671	*/
9672	if ( rcStrict == VINF_SUCCESS
9673	&& VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF \| VMCPU_FF_VMX_PREEMPT_TIMER
9674	\| VMCPU_FF_VMX_INT_WINDOW \| VMCPU_FF_VMX_NMI_WINDOW))
9675	rcStrict = iemHandleNestedInstructionBoundraryFFs(pVCpu, rcStrict);
9676	#endif
9677
9678	/* Execute the next instruction as well if a cli, pop ss or
9679	mov ss, Gr has just completed successfully. */
9680	if ( fExecuteInhibit
9681	&& rcStrict == VINF_SUCCESS
9682	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
9683	&& EMIsInhibitInterruptsActive(pVCpu))
9684	{
9685	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fBypassHandlers, pVCpu->iem.s.fDisregardLock);
9686	if (rcStrict == VINF_SUCCESS)
9687	{
9688	#ifdef LOG_ENABLED
9689	iemLogCurInstr(pVCpu, false, pszFunction);
9690	#endif
9691	#ifdef IEM_WITH_SETJMP
9692	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
9693	if ((rcStrict = setjmp(JmpBuf)) == 0)
9694	{
9695	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9696	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9697	}
9698	else
9699	pVCpu->iem.s.cLongJumps++;
9700	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
9701	#else
9702	IEM_OPCODE_GET_NEXT_U8(&b);
9703	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9704	#endif
9705	if (rcStrict == VINF_SUCCESS)
9706	pVCpu->iem.s.cInstructions++;
9707	if (pVCpu->iem.s.cActiveMappings > 0)
9708	{
9709	Assert(rcStrict != VINF_SUCCESS);
9710	iemMemRollback(pVCpu);
9711	}
9712	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9713	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9714	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9715	}
9716	else if (pVCpu->iem.s.cActiveMappings > 0)
9717	iemMemRollback(pVCpu);
9718	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); /* hope this is correct for all exceptional cases... */
9719	}
9720
9721	/*
9722	* Return value fiddling, statistics and sanity assertions.
9723	*/
9724	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9725
9726	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
9727	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
9728	return rcStrict;
9729	}
9730
9731
9732	/**
9733	* Execute one instruction.
9734	*
9735	* @return Strict VBox status code.
9736	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9737	*/
9738	VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu)
9739	{
9740	AssertCompile(sizeof(pVCpu->iem.s) <= sizeof(pVCpu->iem.padding)); /* (tstVMStruct can't do it's job w/o instruction stats) */
9741	#ifdef LOG_ENABLED
9742	iemLogCurInstr(pVCpu, true, "IEMExecOne");
9743	#endif
9744
9745	/*
9746	* Do the decoding and emulation.
9747	*/
9748	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9749	if (rcStrict == VINF_SUCCESS)
9750	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");
9751	else if (pVCpu->iem.s.cActiveMappings > 0)
9752	iemMemRollback(pVCpu);
9753
9754	if (rcStrict != VINF_SUCCESS)
9755	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9756	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9757	return rcStrict;
9758	}
9759
9760
9761	VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
9762	{
9763	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9764
9765	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9766	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9767	if (rcStrict == VINF_SUCCESS)
9768	{
9769	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx");
9770	if (pcbWritten)
9771	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9772	}
9773	else if (pVCpu->iem.s.cActiveMappings > 0)
9774	iemMemRollback(pVCpu);
9775
9776	return rcStrict;
9777	}
9778
9779
9780	VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9781	const void *pvOpcodeBytes, size_t cbOpcodeBytes)
9782	{
9783	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9784
9785	VBOXSTRICTRC rcStrict;
9786	if ( cbOpcodeBytes
9787	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9788	{
9789	iemInitDecoder(pVCpu, false, false);
9790	#ifdef IEM_WITH_CODE_TLB
9791	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9792	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9793	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9794	pVCpu->iem.s.offCurInstrStart = 0;
9795	pVCpu->iem.s.offInstrNextByte = 0;
9796	#else
9797	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9798	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9799	#endif
9800	rcStrict = VINF_SUCCESS;
9801	}
9802	else
9803	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9804	if (rcStrict == VINF_SUCCESS)
9805	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");
9806	else if (pVCpu->iem.s.cActiveMappings > 0)
9807	iemMemRollback(pVCpu);
9808
9809	return rcStrict;
9810	}
9811
9812
9813	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
9814	{
9815	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9816
9817	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9818	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9819	if (rcStrict == VINF_SUCCESS)
9820	{
9821	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx");
9822	if (pcbWritten)
9823	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9824	}
9825	else if (pVCpu->iem.s.cActiveMappings > 0)
9826	iemMemRollback(pVCpu);
9827
9828	return rcStrict;
9829	}
9830
9831
9832	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9833	const void *pvOpcodeBytes, size_t cbOpcodeBytes)
9834	{
9835	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9836
9837	VBOXSTRICTRC rcStrict;
9838	if ( cbOpcodeBytes
9839	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9840	{
9841	iemInitDecoder(pVCpu, true, false);
9842	#ifdef IEM_WITH_CODE_TLB
9843	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9844	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9845	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9846	pVCpu->iem.s.offCurInstrStart = 0;
9847	pVCpu->iem.s.offInstrNextByte = 0;
9848	#else
9849	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9850	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9851	#endif
9852	rcStrict = VINF_SUCCESS;
9853	}
9854	else
9855	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9856	if (rcStrict == VINF_SUCCESS)
9857	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");
9858	else if (pVCpu->iem.s.cActiveMappings > 0)
9859	iemMemRollback(pVCpu);
9860
9861	return rcStrict;
9862	}
9863
9864
9865	/**
9866	* For debugging DISGetParamSize, may come in handy.
9867	*
9868	* @returns Strict VBox status code.
9869	* @param pVCpu The cross context virtual CPU structure of the
9870	* calling EMT.
9871	* @param pCtxCore The context core structure.
9872	* @param OpcodeBytesPC The PC of the opcode bytes.
9873	* @param pvOpcodeBytes Prefeched opcode bytes.
9874	* @param cbOpcodeBytes Number of prefetched bytes.
9875	* @param pcbWritten Where to return the number of bytes written.
9876	* Optional.
9877	*/
9878	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPCWritten(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9879	const void *pvOpcodeBytes, size_t cbOpcodeBytes,
9880	uint32_t *pcbWritten)
9881	{
9882	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9883
9884	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9885	VBOXSTRICTRC rcStrict;
9886	if ( cbOpcodeBytes
9887	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9888	{
9889	iemInitDecoder(pVCpu, true, false);
9890	#ifdef IEM_WITH_CODE_TLB
9891	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9892	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9893	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9894	pVCpu->iem.s.offCurInstrStart = 0;
9895	pVCpu->iem.s.offInstrNextByte = 0;
9896	#else
9897	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9898	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9899	#endif
9900	rcStrict = VINF_SUCCESS;
9901	}
9902	else
9903	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9904	if (rcStrict == VINF_SUCCESS)
9905	{
9906	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPCWritten");
9907	if (pcbWritten)
9908	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9909	}
9910	else if (pVCpu->iem.s.cActiveMappings > 0)
9911	iemMemRollback(pVCpu);
9912
9913	return rcStrict;
9914	}
9915
9916
9917	/**
9918	* For handling split cacheline lock operations when the host has split-lock
9919	* detection enabled.
9920	*
9921	* This will cause the interpreter to disregard the lock prefix and implicit
9922	* locking (xchg).
9923	*
9924	* @returns Strict VBox status code.
9925	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9926	*/
9927	VMMDECL(VBOXSTRICTRC) IEMExecOneIgnoreLock(PVMCPUCC pVCpu)
9928	{
9929	/*
9930	* Do the decoding and emulation.
9931	*/
9932	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, true /fDisregardLock/);
9933	if (rcStrict == VINF_SUCCESS)
9934	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneIgnoreLock");
9935	else if (pVCpu->iem.s.cActiveMappings > 0)
9936	iemMemRollback(pVCpu);
9937
9938	if (rcStrict != VINF_SUCCESS)
9939	LogFlow(("IEMExecOneIgnoreLock: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9940	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9941	return rcStrict;
9942	}
9943
9944
9945	VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions)
9946	{
9947	uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;
9948	AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate));
9949
9950	/*
9951	* See if there is an interrupt pending in TRPM, inject it if we can.
9952	*/
9953	/** @todo Can we centralize this under CPUMCanInjectInterrupt()? */
9954	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9955	bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);
9956	if (fIntrEnabled)
9957	{
9958	if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu)))
9959	fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
9960	else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
9961	fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu));
9962	else
9963	{
9964	Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
9965	fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu));
9966	}
9967	}
9968	#else
9969	bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
9970	#endif
9971
9972	/** @todo What if we are injecting an exception and not an interrupt? Is that
9973	* possible here? For now we assert it is indeed only an interrupt. */
9974	if ( fIntrEnabled
9975	&& TRPMHasTrap(pVCpu)
9976	&& EMGetInhibitInterruptsPC(pVCpu) != pVCpu->cpum.GstCtx.rip)
9977	{
9978	uint8_t u8TrapNo;
9979	TRPMEVENT enmType;
9980	uint32_t uErrCode;
9981	RTGCPTR uCr2;
9982	int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /* pu8InstLen /, NULL / fIcebp */);
9983	AssertRC(rc2);
9984	Assert(enmType == TRPM_HARDWARE_INT);
9985	VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /* cbInstr */);
9986	TRPMResetTrap(pVCpu);
9987	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9988	/* Injecting an event may cause a VM-exit. */
9989	if ( rcStrict != VINF_SUCCESS
9990	&& rcStrict != VINF_IEM_RAISED_XCPT)
9991	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
9992	#else
9993	NOREF(rcStrict);
9994	#endif
9995	}
9996
9997	/*
9998	* Initial decoder init w/ prefetch, then setup setjmp.
9999	*/
10000	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
10001	if (rcStrict == VINF_SUCCESS)
10002	{
10003	#ifdef IEM_WITH_SETJMP
10004	jmp_buf JmpBuf;
10005	jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
10006	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
10007	pVCpu->iem.s.cActiveMappings = 0;
10008	if ((rcStrict = setjmp(JmpBuf)) == 0)
10009	#endif
10010	{
10011	/*
10012	* The run loop. We limit ourselves to 4096 instructions right now.
10013	*/
10014	uint32_t cMaxInstructionsGccStupidity = cMaxInstructions;
10015	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10016	for (;;)
10017	{
10018	/*
10019	* Log the state.
10020	*/
10021	#ifdef LOG_ENABLED
10022	iemLogCurInstr(pVCpu, true, "IEMExecLots");
10023	#endif
10024
10025	/*
10026	* Do the decoding and emulation.
10027	*/
10028	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
10029	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
10030	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10031	{
10032	Assert(pVCpu->iem.s.cActiveMappings == 0);
10033	pVCpu->iem.s.cInstructions++;
10034	if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
10035	{
10036	uint64_t fCpu = pVCpu->fLocalForcedActions
10037	& ( VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
10038	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
10039	\| VMCPU_FF_TLB_FLUSH
10040	\| VMCPU_FF_INHIBIT_INTERRUPTS
10041	\| VMCPU_FF_BLOCK_NMIS
10042	\| VMCPU_FF_UNHALT ));
10043
10044	if (RT_LIKELY( ( !fCpu
10045	\|\| ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
10046	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
10047	&& !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
10048	{
10049	if (cMaxInstructionsGccStupidity-- > 0)
10050	{
10051	/* Poll timers every now an then according to the caller's specs. */
10052	if ( (cMaxInstructionsGccStupidity & cPollRate) != 0
10053	\|\| !TMTimerPollBool(pVM, pVCpu))
10054	{
10055	Assert(pVCpu->iem.s.cActiveMappings == 0);
10056	iemReInitDecoder(pVCpu);
10057	continue;
10058	}
10059	}
10060	}
10061	}
10062	Assert(pVCpu->iem.s.cActiveMappings == 0);
10063	}
10064	else if (pVCpu->iem.s.cActiveMappings > 0)
10065	iemMemRollback(pVCpu);
10066	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10067	break;
10068	}
10069	}
10070	#ifdef IEM_WITH_SETJMP
10071	else
10072	{
10073	if (pVCpu->iem.s.cActiveMappings > 0)
10074	iemMemRollback(pVCpu);
10075	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10076	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10077	# endif
10078	pVCpu->iem.s.cLongJumps++;
10079	}
10080	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
10081	#endif
10082
10083	/*
10084	* Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
10085	*/
10086	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
10087	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
10088	}
10089	else
10090	{
10091	if (pVCpu->iem.s.cActiveMappings > 0)
10092	iemMemRollback(pVCpu);
10093
10094	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10095	/*
10096	* When a nested-guest causes an exception intercept (e.g. #PF) when fetching
10097	* code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
10098	*/
10099	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10100	#endif
10101	}
10102
10103	/*
10104	* Maybe re-enter raw-mode and log.
10105	*/
10106	if (rcStrict != VINF_SUCCESS)
10107	LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10108	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10109	if (pcInstructions)
10110	*pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;
10111	return rcStrict;
10112	}
10113
10114
10115	/**
10116	* Interface used by EMExecuteExec, does exit statistics and limits.
10117	*
10118	* @returns Strict VBox status code.
10119	* @param pVCpu The cross context virtual CPU structure.
10120	* @param fWillExit To be defined.
10121	* @param cMinInstructions Minimum number of instructions to execute before checking for FFs.
10122	* @param cMaxInstructions Maximum number of instructions to execute.
10123	* @param cMaxInstructionsWithoutExits
10124	* The max number of instructions without exits.
10125	* @param pStats Where to return statistics.
10126	*/
10127	VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,
10128	uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)
10129	{
10130	NOREF(fWillExit); /** @todo define flexible exit crits */
10131
10132	/*
10133	* Initialize return stats.
10134	*/
10135	pStats->cInstructions = 0;
10136	pStats->cExits = 0;
10137	pStats->cMaxExitDistance = 0;
10138	pStats->cReserved = 0;
10139
10140	/*
10141	* Initial decoder init w/ prefetch, then setup setjmp.
10142	*/
10143	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
10144	if (rcStrict == VINF_SUCCESS)
10145	{
10146	#ifdef IEM_WITH_SETJMP
10147	jmp_buf JmpBuf;
10148	jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
10149	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
10150	pVCpu->iem.s.cActiveMappings = 0;
10151	if ((rcStrict = setjmp(JmpBuf)) == 0)
10152	#endif
10153	{
10154	#ifdef IN_RING0
10155	bool const fCheckPreemptionPending = !RTThreadPreemptIsPossible() \|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
10156	#endif
10157	uint32_t cInstructionSinceLastExit = 0;
10158
10159	/*
10160	* The run loop. We limit ourselves to 4096 instructions right now.
10161	*/
10162	PVM pVM = pVCpu->CTX_SUFF(pVM);
10163	for (;;)
10164	{
10165	/*
10166	* Log the state.
10167	*/
10168	#ifdef LOG_ENABLED
10169	iemLogCurInstr(pVCpu, true, "IEMExecForExits");
10170	#endif
10171
10172	/*
10173	* Do the decoding and emulation.
10174	*/
10175	uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;
10176
10177	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
10178	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
10179
10180	if ( cPotentialExits != pVCpu->iem.s.cPotentialExits
10181	&& cInstructionSinceLastExit > 0 /* don't count the first */ )
10182	{
10183	pStats->cExits += 1;
10184	if (cInstructionSinceLastExit > pStats->cMaxExitDistance)
10185	pStats->cMaxExitDistance = cInstructionSinceLastExit;
10186	cInstructionSinceLastExit = 0;
10187	}
10188
10189	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10190	{
10191	Assert(pVCpu->iem.s.cActiveMappings == 0);
10192	pVCpu->iem.s.cInstructions++;
10193	pStats->cInstructions++;
10194	cInstructionSinceLastExit++;
10195	if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
10196	{
10197	uint64_t fCpu = pVCpu->fLocalForcedActions
10198	& ( VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
10199	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
10200	\| VMCPU_FF_TLB_FLUSH
10201	\| VMCPU_FF_INHIBIT_INTERRUPTS
10202	\| VMCPU_FF_BLOCK_NMIS
10203	\| VMCPU_FF_UNHALT ));
10204
10205	if (RT_LIKELY( ( ( !fCpu
10206	\|\| ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
10207	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))
10208	&& !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )
10209	\|\| pStats->cInstructions < cMinInstructions))
10210	{
10211	if (pStats->cInstructions < cMaxInstructions)
10212	{
10213	if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)
10214	{
10215	#ifdef IN_RING0
10216	if ( !fCheckPreemptionPending
10217	\|\| !RTThreadPreemptIsPending(NIL_RTTHREAD))
10218	#endif
10219	{
10220	Assert(pVCpu->iem.s.cActiveMappings == 0);
10221	iemReInitDecoder(pVCpu);
10222	continue;
10223	}
10224	#ifdef IN_RING0
10225	rcStrict = VINF_EM_RAW_INTERRUPT;
10226	break;
10227	#endif
10228	}
10229	}
10230	}
10231	Assert(!(fCpu & VMCPU_FF_IEM));
10232	}
10233	Assert(pVCpu->iem.s.cActiveMappings == 0);
10234	}
10235	else if (pVCpu->iem.s.cActiveMappings > 0)
10236	iemMemRollback(pVCpu);
10237	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10238	break;
10239	}
10240	}
10241	#ifdef IEM_WITH_SETJMP
10242	else
10243	{
10244	if (pVCpu->iem.s.cActiveMappings > 0)
10245	iemMemRollback(pVCpu);
10246	pVCpu->iem.s.cLongJumps++;
10247	}
10248	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
10249	#endif
10250
10251	/*
10252	* Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
10253	*/
10254	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
10255	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
10256	}
10257	else
10258	{
10259	if (pVCpu->iem.s.cActiveMappings > 0)
10260	iemMemRollback(pVCpu);
10261
10262	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10263	/*
10264	* When a nested-guest causes an exception intercept (e.g. #PF) when fetching
10265	* code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
10266	*/
10267	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10268	#endif
10269	}
10270
10271	/*
10272	* Maybe re-enter raw-mode and log.
10273	*/
10274	if (rcStrict != VINF_SUCCESS)
10275	LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",
10276	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,
10277	pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));
10278	return rcStrict;
10279	}
10280
10281
10282	/**
10283	* Injects a trap, fault, abort, software interrupt or external interrupt.
10284	*
10285	* The parameter list matches TRPMQueryTrapAll pretty closely.
10286	*
10287	* @returns Strict VBox status code.
10288	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10289	* @param u8TrapNo The trap number.
10290	* @param enmType What type is it (trap/fault/abort), software
10291	* interrupt or hardware interrupt.
10292	* @param uErrCode The error code if applicable.
10293	* @param uCr2 The CR2 value if applicable.
10294	* @param cbInstr The instruction length (only relevant for
10295	* software interrupts).
10296	*/
10297	VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
10298	uint8_t cbInstr)
10299	{
10300	iemInitDecoder(pVCpu, false, false);
10301	#ifdef DBGFTRACE_ENABLED
10302	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
10303	u8TrapNo, enmType, uErrCode, uCr2);
10304	#endif
10305
10306	uint32_t fFlags;
10307	switch (enmType)
10308	{
10309	case TRPM_HARDWARE_INT:
10310	Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
10311	fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
10312	uErrCode = uCr2 = 0;
10313	break;
10314
10315	case TRPM_SOFTWARE_INT:
10316	Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
10317	fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
10318	uErrCode = uCr2 = 0;
10319	break;
10320
10321	case TRPM_TRAP:
10322	Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
10323	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
10324	if (u8TrapNo == X86_XCPT_PF)
10325	fFlags \|= IEM_XCPT_FLAGS_CR2;
10326	switch (u8TrapNo)
10327	{
10328	case X86_XCPT_DF:
10329	case X86_XCPT_TS:
10330	case X86_XCPT_NP:
10331	case X86_XCPT_SS:
10332	case X86_XCPT_PF:
10333	case X86_XCPT_AC:
10334	case X86_XCPT_GP:
10335	fFlags \|= IEM_XCPT_FLAGS_ERR;
10336	break;
10337	}
10338	break;
10339
10340	IEM_NOT_REACHED_DEFAULT_CASE_RET();
10341	}
10342
10343	VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
10344
10345	if (pVCpu->iem.s.cActiveMappings > 0)
10346	iemMemRollback(pVCpu);
10347
10348	return rcStrict;
10349	}
10350
10351
10352	/**
10353	* Injects the active TRPM event.
10354	*
10355	* @returns Strict VBox status code.
10356	* @param pVCpu The cross context virtual CPU structure.
10357	*/
10358	VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu)
10359	{
10360	#ifndef IEM_IMPLEMENTS_TASKSWITCH
10361	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
10362	#else
10363	uint8_t u8TrapNo;
10364	TRPMEVENT enmType;
10365	uint32_t uErrCode;
10366	RTGCUINTPTR uCr2;
10367	uint8_t cbInstr;
10368	int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */);
10369	if (RT_FAILURE(rc))
10370	return rc;
10371
10372	/** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle
10373	* ICEBP \#DB injection as a special case. */
10374	VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
10375	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
10376	if (rcStrict == VINF_SVM_VMEXIT)
10377	rcStrict = VINF_SUCCESS;
10378	#endif
10379	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10380	if (rcStrict == VINF_VMX_VMEXIT)
10381	rcStrict = VINF_SUCCESS;
10382	#endif
10383	/** @todo Are there any other codes that imply the event was successfully
10384	* delivered to the guest? See @bugref{6607}. */
10385	if ( rcStrict == VINF_SUCCESS
10386	\|\| rcStrict == VINF_IEM_RAISED_XCPT)
10387	TRPMResetTrap(pVCpu);
10388
10389	return rcStrict;
10390	#endif
10391	}
10392
10393
10394	VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
10395	{
10396	RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
10397	return VERR_NOT_IMPLEMENTED;
10398	}
10399
10400
10401	VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
10402	{
10403	RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
10404	return VERR_NOT_IMPLEMENTED;
10405	}
10406
10407
10408	#if 0 /* The IRET-to-v8086 mode in PATM is very optimistic, so I don't dare do this yet. */
10409	/**
10410	* Executes a IRET instruction with default operand size.
10411	*
10412	* This is for PATM.
10413	*
10414	* @returns VBox status code.
10415	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10416	* @param pCtxCore The register frame.
10417	*/
10418	VMM_INT_DECL(int) IEMExecInstr_iret(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore)
10419	{
10420	PCPUMCTX pCtx = IEM_GET_CTX(pVCpu);
10421
10422	iemCtxCoreToCtx(pCtx, pCtxCore);
10423	iemInitDecoder(pVCpu);
10424	VBOXSTRICTRC rcStrict = iemCImpl_iret(pVCpu, 1, pVCpu->iem.s.enmDefOpSize);
10425	if (rcStrict == VINF_SUCCESS)
10426	iemCtxToCtxCore(pCtxCore, pCtx);
10427	else
10428	LogFlow(("IEMExecInstr_iret: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10429	pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10430	return rcStrict;
10431	}
10432	#endif
10433
10434
10435	/**
10436	* Interface for HM and EM for executing string I/O OUT (write) instructions.
10437	*
10438	* This API ASSUMES that the caller has already verified that the guest code is
10439	* allowed to access the I/O port. (The I/O port is in the DX register in the
10440	* guest state.)
10441	*
10442	* @returns Strict VBox status code.
10443	* @param pVCpu The cross context virtual CPU structure.
10444	* @param cbValue The size of the I/O port access (1, 2, or 4).
10445	* @param enmAddrMode The addressing mode.
10446	* @param fRepPrefix Indicates whether a repeat prefix is used
10447	* (doesn't matter which for this instruction).
10448	* @param cbInstr The instruction length in bytes.
10449	* @param iEffSeg The effective segment address.
10450	* @param fIoChecked Whether the access to the I/O port has been
10451	* checked or not. It's typically checked in the
10452	* HM scenario.
10453	*/
10454	VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
10455	bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked)
10456	{
10457	AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
10458	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10459
10460	/*
10461	* State init.
10462	*/
10463	iemInitExec(pVCpu, false /fBypassHandlers/);
10464
10465	/*
10466	* Switch orgy for getting to the right handler.
10467	*/
10468	VBOXSTRICTRC rcStrict;
10469	if (fRepPrefix)
10470	{
10471	switch (enmAddrMode)
10472	{
10473	case IEMMODE_16BIT:
10474	switch (cbValue)
10475	{
10476	case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10477	case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10478	case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10479	default:
10480	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10481	}
10482	break;
10483
10484	case IEMMODE_32BIT:
10485	switch (cbValue)
10486	{
10487	case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10488	case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10489	case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10490	default:
10491	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10492	}
10493	break;
10494
10495	case IEMMODE_64BIT:
10496	switch (cbValue)
10497	{
10498	case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10499	case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10500	case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10501	default:
10502	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10503	}
10504	break;
10505
10506	default:
10507	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10508	}
10509	}
10510	else
10511	{
10512	switch (enmAddrMode)
10513	{
10514	case IEMMODE_16BIT:
10515	switch (cbValue)
10516	{
10517	case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10518	case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10519	case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10520	default:
10521	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10522	}
10523	break;
10524
10525	case IEMMODE_32BIT:
10526	switch (cbValue)
10527	{
10528	case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10529	case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10530	case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10531	default:
10532	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10533	}
10534	break;
10535
10536	case IEMMODE_64BIT:
10537	switch (cbValue)
10538	{
10539	case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10540	case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10541	case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10542	default:
10543	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10544	}
10545	break;
10546
10547	default:
10548	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10549	}
10550	}
10551
10552	if (pVCpu->iem.s.cActiveMappings)
10553	iemMemRollback(pVCpu);
10554
10555	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10556	}
10557
10558
10559	/**
10560	* Interface for HM and EM for executing string I/O IN (read) instructions.
10561	*
10562	* This API ASSUMES that the caller has already verified that the guest code is
10563	* allowed to access the I/O port. (The I/O port is in the DX register in the
10564	* guest state.)
10565	*
10566	* @returns Strict VBox status code.
10567	* @param pVCpu The cross context virtual CPU structure.
10568	* @param cbValue The size of the I/O port access (1, 2, or 4).
10569	* @param enmAddrMode The addressing mode.
10570	* @param fRepPrefix Indicates whether a repeat prefix is used
10571	* (doesn't matter which for this instruction).
10572	* @param cbInstr The instruction length in bytes.
10573	* @param fIoChecked Whether the access to the I/O port has been
10574	* checked or not. It's typically checked in the
10575	* HM scenario.
10576	*/
10577	VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
10578	bool fRepPrefix, uint8_t cbInstr, bool fIoChecked)
10579	{
10580	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10581
10582	/*
10583	* State init.
10584	*/
10585	iemInitExec(pVCpu, false /fBypassHandlers/);
10586
10587	/*
10588	* Switch orgy for getting to the right handler.
10589	*/
10590	VBOXSTRICTRC rcStrict;
10591	if (fRepPrefix)
10592	{
10593	switch (enmAddrMode)
10594	{
10595	case IEMMODE_16BIT:
10596	switch (cbValue)
10597	{
10598	case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
10599	case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
10600	case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
10601	default:
10602	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10603	}
10604	break;
10605
10606	case IEMMODE_32BIT:
10607	switch (cbValue)
10608	{
10609	case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
10610	case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
10611	case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
10612	default:
10613	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10614	}
10615	break;
10616
10617	case IEMMODE_64BIT:
10618	switch (cbValue)
10619	{
10620	case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
10621	case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
10622	case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
10623	default:
10624	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10625	}
10626	break;
10627
10628	default:
10629	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10630	}
10631	}
10632	else
10633	{
10634	switch (enmAddrMode)
10635	{
10636	case IEMMODE_16BIT:
10637	switch (cbValue)
10638	{
10639	case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
10640	case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
10641	case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
10642	default:
10643	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10644	}
10645	break;
10646
10647	case IEMMODE_32BIT:
10648	switch (cbValue)
10649	{
10650	case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
10651	case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
10652	case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
10653	default:
10654	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10655	}
10656	break;
10657
10658	case IEMMODE_64BIT:
10659	switch (cbValue)
10660	{
10661	case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
10662	case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
10663	case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
10664	default:
10665	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10666	}
10667	break;
10668
10669	default:
10670	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10671	}
10672	}
10673
10674	if ( pVCpu->iem.s.cActiveMappings == 0
10675	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
10676	{ /* likely */ }
10677	else
10678	{
10679	AssertMsg(!IOM_SUCCESS(rcStrict), ("%#x\n", VBOXSTRICTRC_VAL(rcStrict)));
10680	iemMemRollback(pVCpu);
10681	}
10682	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10683	}
10684
10685
10686	/**
10687	* Interface for rawmode to write execute an OUT instruction.
10688	*
10689	* @returns Strict VBox status code.
10690	* @param pVCpu The cross context virtual CPU structure.
10691	* @param cbInstr The instruction length in bytes.
10692	* @param u16Port The port to read.
10693	* @param fImm Whether the port is specified using an immediate operand or
10694	* using the implicit DX register.
10695	* @param cbReg The register size.
10696	*
10697	* @remarks In ring-0 not all of the state needs to be synced in.
10698	*/
10699	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
10700	{
10701	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10702	Assert(cbReg <= 4 && cbReg != 3);
10703
10704	iemInitExec(pVCpu, false /fBypassHandlers/);
10705	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_out, u16Port, fImm, cbReg);
10706	Assert(!pVCpu->iem.s.cActiveMappings);
10707	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10708	}
10709
10710
10711	/**
10712	* Interface for rawmode to write execute an IN instruction.
10713	*
10714	* @returns Strict VBox status code.
10715	* @param pVCpu The cross context virtual CPU structure.
10716	* @param cbInstr The instruction length in bytes.
10717	* @param u16Port The port to read.
10718	* @param fImm Whether the port is specified using an immediate operand or
10719	* using the implicit DX.
10720	* @param cbReg The register size.
10721	*/
10722	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
10723	{
10724	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10725	Assert(cbReg <= 4 && cbReg != 3);
10726
10727	iemInitExec(pVCpu, false /fBypassHandlers/);
10728	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_in, u16Port, fImm, cbReg);
10729	Assert(!pVCpu->iem.s.cActiveMappings);
10730	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10731	}
10732
10733
10734	/**
10735	* Interface for HM and EM to write to a CRx register.
10736	*
10737	* @returns Strict VBox status code.
10738	* @param pVCpu The cross context virtual CPU structure.
10739	* @param cbInstr The instruction length in bytes.
10740	* @param iCrReg The control register number (destination).
10741	* @param iGReg The general purpose register number (source).
10742	*
10743	* @remarks In ring-0 not all of the state needs to be synced in.
10744	*/
10745	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg)
10746	{
10747	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10748	Assert(iCrReg < 16);
10749	Assert(iGReg < 16);
10750
10751	iemInitExec(pVCpu, false /fBypassHandlers/);
10752	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg);
10753	Assert(!pVCpu->iem.s.cActiveMappings);
10754	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10755	}
10756
10757
10758	/**
10759	* Interface for HM and EM to read from a CRx register.
10760	*
10761	* @returns Strict VBox status code.
10762	* @param pVCpu The cross context virtual CPU structure.
10763	* @param cbInstr The instruction length in bytes.
10764	* @param iGReg The general purpose register number (destination).
10765	* @param iCrReg The control register number (source).
10766	*
10767	* @remarks In ring-0 not all of the state needs to be synced in.
10768	*/
10769	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
10770	{
10771	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10772	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
10773	\| CPUMCTX_EXTRN_APIC_TPR);
10774	Assert(iCrReg < 16);
10775	Assert(iGReg < 16);
10776
10777	iemInitExec(pVCpu, false /fBypassHandlers/);
10778	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg);
10779	Assert(!pVCpu->iem.s.cActiveMappings);
10780	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10781	}
10782
10783
10784	/**
10785	* Interface for HM and EM to clear the CR0[TS] bit.
10786	*
10787	* @returns Strict VBox status code.
10788	* @param pVCpu The cross context virtual CPU structure.
10789	* @param cbInstr The instruction length in bytes.
10790	*
10791	* @remarks In ring-0 not all of the state needs to be synced in.
10792	*/
10793	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPUCC pVCpu, uint8_t cbInstr)
10794	{
10795	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10796
10797	iemInitExec(pVCpu, false /fBypassHandlers/);
10798	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts);
10799	Assert(!pVCpu->iem.s.cActiveMappings);
10800	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10801	}
10802
10803
10804	/**
10805	* Interface for HM and EM to emulate the LMSW instruction (loads CR0).
10806	*
10807	* @returns Strict VBox status code.
10808	* @param pVCpu The cross context virtual CPU structure.
10809	* @param cbInstr The instruction length in bytes.
10810	* @param uValue The value to load into CR0.
10811	* @param GCPtrEffDst The guest-linear address if the LMSW instruction has a
10812	* memory operand. Otherwise pass NIL_RTGCPTR.
10813	*
10814	* @remarks In ring-0 not all of the state needs to be synced in.
10815	*/
10816	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uValue, RTGCPTR GCPtrEffDst)
10817	{
10818	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10819
10820	iemInitExec(pVCpu, false /fBypassHandlers/);
10821	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_lmsw, uValue, GCPtrEffDst);
10822	Assert(!pVCpu->iem.s.cActiveMappings);
10823	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10824	}
10825
10826
10827	/**
10828	* Interface for HM and EM to emulate the XSETBV instruction (loads XCRx).
10829	*
10830	* Takes input values in ecx and edx:eax of the CPU context of the calling EMT.
10831	*
10832	* @returns Strict VBox status code.
10833	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10834	* @param cbInstr The instruction length in bytes.
10835	* @remarks In ring-0 not all of the state needs to be synced in.
10836	* @thread EMT(pVCpu)
10837	*/
10838	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPUCC pVCpu, uint8_t cbInstr)
10839	{
10840	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10841
10842	iemInitExec(pVCpu, false /fBypassHandlers/);
10843	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv);
10844	Assert(!pVCpu->iem.s.cActiveMappings);
10845	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10846	}
10847
10848
10849	/**
10850	* Interface for HM and EM to emulate the WBINVD instruction.
10851	*
10852	* @returns Strict VBox status code.
10853	* @param pVCpu The cross context virtual CPU structure.
10854	* @param cbInstr The instruction length in bytes.
10855	*
10856	* @remarks In ring-0 not all of the state needs to be synced in.
10857	*/
10858	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPUCC pVCpu, uint8_t cbInstr)
10859	{
10860	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10861
10862	iemInitExec(pVCpu, false /fBypassHandlers/);
10863	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd);
10864	Assert(!pVCpu->iem.s.cActiveMappings);
10865	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10866	}
10867
10868
10869	/**
10870	* Interface for HM and EM to emulate the INVD instruction.
10871	*
10872	* @returns Strict VBox status code.
10873	* @param pVCpu The cross context virtual CPU structure.
10874	* @param cbInstr The instruction length in bytes.
10875	*
10876	* @remarks In ring-0 not all of the state needs to be synced in.
10877	*/
10878	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPUCC pVCpu, uint8_t cbInstr)
10879	{
10880	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10881
10882	iemInitExec(pVCpu, false /fBypassHandlers/);
10883	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd);
10884	Assert(!pVCpu->iem.s.cActiveMappings);
10885	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10886	}
10887
10888
10889	/**
10890	* Interface for HM and EM to emulate the INVLPG instruction.
10891	*
10892	* @returns Strict VBox status code.
10893	* @retval VINF_PGM_SYNC_CR3
10894	*
10895	* @param pVCpu The cross context virtual CPU structure.
10896	* @param cbInstr The instruction length in bytes.
10897	* @param GCPtrPage The effective address of the page to invalidate.
10898	*
10899	* @remarks In ring-0 not all of the state needs to be synced in.
10900	*/
10901	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage)
10902	{
10903	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10904
10905	iemInitExec(pVCpu, false /fBypassHandlers/);
10906	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage);
10907	Assert(!pVCpu->iem.s.cActiveMappings);
10908	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10909	}
10910
10911
10912	/**
10913	* Interface for HM and EM to emulate the INVPCID instruction.
10914	*
10915	* @returns Strict VBox status code.
10916	* @retval VINF_PGM_SYNC_CR3
10917	*
10918	* @param pVCpu The cross context virtual CPU structure.
10919	* @param cbInstr The instruction length in bytes.
10920	* @param iEffSeg The effective segment register.
10921	* @param GCPtrDesc The effective address of the INVPCID descriptor.
10922	* @param uType The invalidation type.
10923	*
10924	* @remarks In ring-0 not all of the state needs to be synced in.
10925	*/
10926	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvpcid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDesc,
10927	uint64_t uType)
10928	{
10929	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 4);
10930
10931	iemInitExec(pVCpu, false /fBypassHandlers/);
10932	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_invpcid, iEffSeg, GCPtrDesc, uType);
10933	Assert(!pVCpu->iem.s.cActiveMappings);
10934	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10935	}
10936
10937
10938	/**
10939	* Interface for HM and EM to emulate the CPUID instruction.
10940	*
10941	* @returns Strict VBox status code.
10942	*
10943	* @param pVCpu The cross context virtual CPU structure.
10944	* @param cbInstr The instruction length in bytes.
10945	*
10946	* @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX.
10947	*/
10948	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPUCC pVCpu, uint8_t cbInstr)
10949	{
10950	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10951	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
10952
10953	iemInitExec(pVCpu, false /fBypassHandlers/);
10954	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid);
10955	Assert(!pVCpu->iem.s.cActiveMappings);
10956	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10957	}
10958
10959
10960	/**
10961	* Interface for HM and EM to emulate the RDPMC instruction.
10962	*
10963	* @returns Strict VBox status code.
10964	*
10965	* @param pVCpu The cross context virtual CPU structure.
10966	* @param cbInstr The instruction length in bytes.
10967	*
10968	* @remarks Not all of the state needs to be synced in.
10969	*/
10970	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPUCC pVCpu, uint8_t cbInstr)
10971	{
10972	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10973	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
10974
10975	iemInitExec(pVCpu, false /fBypassHandlers/);
10976	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc);
10977	Assert(!pVCpu->iem.s.cActiveMappings);
10978	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10979	}
10980
10981
10982	/**
10983	* Interface for HM and EM to emulate the RDTSC instruction.
10984	*
10985	* @returns Strict VBox status code.
10986	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
10987	*
10988	* @param pVCpu The cross context virtual CPU structure.
10989	* @param cbInstr The instruction length in bytes.
10990	*
10991	* @remarks Not all of the state needs to be synced in.
10992	*/
10993	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPUCC pVCpu, uint8_t cbInstr)
10994	{
10995	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10996	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
10997
10998	iemInitExec(pVCpu, false /fBypassHandlers/);
10999	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc);
11000	Assert(!pVCpu->iem.s.cActiveMappings);
11001	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11002	}
11003
11004
11005	/**
11006	* Interface for HM and EM to emulate the RDTSCP instruction.
11007	*
11008	* @returns Strict VBox status code.
11009	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11010	*
11011	* @param pVCpu The cross context virtual CPU structure.
11012	* @param cbInstr The instruction length in bytes.
11013	*
11014	* @remarks Not all of the state needs to be synced in. Recommended
11015	* to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call.
11016	*/
11017	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPUCC pVCpu, uint8_t cbInstr)
11018	{
11019	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11020	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
11021
11022	iemInitExec(pVCpu, false /fBypassHandlers/);
11023	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp);
11024	Assert(!pVCpu->iem.s.cActiveMappings);
11025	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11026	}
11027
11028
11029	/**
11030	* Interface for HM and EM to emulate the RDMSR instruction.
11031	*
11032	* @returns Strict VBox status code.
11033	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11034	*
11035	* @param pVCpu The cross context virtual CPU structure.
11036	* @param cbInstr The instruction length in bytes.
11037	*
11038	* @remarks Not all of the state needs to be synced in. Requires RCX and
11039	* (currently) all MSRs.
11040	*/
11041	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
11042	{
11043	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11044	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_ALL_MSRS);
11045
11046	iemInitExec(pVCpu, false /fBypassHandlers/);
11047	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr);
11048	Assert(!pVCpu->iem.s.cActiveMappings);
11049	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11050	}
11051
11052
11053	/**
11054	* Interface for HM and EM to emulate the WRMSR instruction.
11055	*
11056	* @returns Strict VBox status code.
11057	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11058	*
11059	* @param pVCpu The cross context virtual CPU structure.
11060	* @param cbInstr The instruction length in bytes.
11061	*
11062	* @remarks Not all of the state needs to be synced in. Requires RCX, RAX, RDX,
11063	* and (currently) all MSRs.
11064	*/
11065	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
11066	{
11067	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11068	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
11069	\| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_ALL_MSRS);
11070
11071	iemInitExec(pVCpu, false /fBypassHandlers/);
11072	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr);
11073	Assert(!pVCpu->iem.s.cActiveMappings);
11074	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11075	}
11076
11077
11078	/**
11079	* Interface for HM and EM to emulate the MONITOR instruction.
11080	*
11081	* @returns Strict VBox status code.
11082	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11083	*
11084	* @param pVCpu The cross context virtual CPU structure.
11085	* @param cbInstr The instruction length in bytes.
11086	*
11087	* @remarks Not all of the state needs to be synced in.
11088	* @remarks ASSUMES the default segment of DS and no segment override prefixes
11089	* are used.
11090	*/
11091	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPUCC pVCpu, uint8_t cbInstr)
11092	{
11093	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11094	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
11095
11096	iemInitExec(pVCpu, false /fBypassHandlers/);
11097	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS);
11098	Assert(!pVCpu->iem.s.cActiveMappings);
11099	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11100	}
11101
11102
11103	/**
11104	* Interface for HM and EM to emulate the MWAIT instruction.
11105	*
11106	* @returns Strict VBox status code.
11107	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11108	*
11109	* @param pVCpu The cross context virtual CPU structure.
11110	* @param cbInstr The instruction length in bytes.
11111	*
11112	* @remarks Not all of the state needs to be synced in.
11113	*/
11114	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPUCC pVCpu, uint8_t cbInstr)
11115	{
11116	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11117	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RAX);
11118
11119	iemInitExec(pVCpu, false /fBypassHandlers/);
11120	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait);
11121	Assert(!pVCpu->iem.s.cActiveMappings);
11122	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11123	}
11124
11125
11126	/**
11127	* Interface for HM and EM to emulate the HLT instruction.
11128	*
11129	* @returns Strict VBox status code.
11130	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11131	*
11132	* @param pVCpu The cross context virtual CPU structure.
11133	* @param cbInstr The instruction length in bytes.
11134	*
11135	* @remarks Not all of the state needs to be synced in.
11136	*/
11137	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPUCC pVCpu, uint8_t cbInstr)
11138	{
11139	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
11140
11141	iemInitExec(pVCpu, false /fBypassHandlers/);
11142	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt);
11143	Assert(!pVCpu->iem.s.cActiveMappings);
11144	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11145	}
11146
11147
11148	/**
11149	* Checks if IEM is in the process of delivering an event (interrupt or
11150	* exception).
11151	*
11152	* @returns true if we're in the process of raising an interrupt or exception,
11153	* false otherwise.
11154	* @param pVCpu The cross context virtual CPU structure.
11155	* @param puVector Where to store the vector associated with the
11156	* currently delivered event, optional.
11157	* @param pfFlags Where to store th event delivery flags (see
11158	* IEM_XCPT_FLAGS_XXX), optional.
11159	* @param puErr Where to store the error code associated with the
11160	* event, optional.
11161	* @param puCr2 Where to store the CR2 associated with the event,
11162	* optional.
11163	* @remarks The caller should check the flags to determine if the error code and
11164	* CR2 are valid for the event.
11165	*/
11166	VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPUCC pVCpu, uint8_t puVector, uint32_t pfFlags, uint32_t puErr, uint64_t puCr2)
11167	{
11168	bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0;
11169	if (fRaisingXcpt)
11170	{
11171	if (puVector)
11172	*puVector = pVCpu->iem.s.uCurXcpt;
11173	if (pfFlags)
11174	*pfFlags = pVCpu->iem.s.fCurXcpt;
11175	if (puErr)
11176	*puErr = pVCpu->iem.s.uCurXcptErr;
11177	if (puCr2)
11178	*puCr2 = pVCpu->iem.s.uCurXcptCr2;
11179	}
11180	return fRaisingXcpt;
11181	}
11182
11183	#ifdef IN_RING3
11184
11185	/**
11186	* Handles the unlikely and probably fatal merge cases.
11187	*
11188	* @returns Merged status code.
11189	* @param rcStrict Current EM status code.
11190	* @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
11191	* with @a rcStrict.
11192	* @param iMemMap The memory mapping index. For error reporting only.
11193	* @param pVCpu The cross context virtual CPU structure of the calling
11194	* thread, for error reporting only.
11195	*/
11196	DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
11197	unsigned iMemMap, PVMCPUCC pVCpu)
11198	{
11199	if (RT_FAILURE_NP(rcStrict))
11200	return rcStrict;
11201
11202	if (RT_FAILURE_NP(rcStrictCommit))
11203	return rcStrictCommit;
11204
11205	if (rcStrict == rcStrictCommit)
11206	return rcStrictCommit;
11207
11208	AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",
11209	VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,
11210	pVCpu->iem.s.aMemMappings[iMemMap].fAccess,
11211	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,
11212	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));
11213	return VERR_IOM_FF_STATUS_IPE;
11214	}
11215
11216
11217	/**
11218	* Helper for IOMR3ProcessForceFlag.
11219	*
11220	* @returns Merged status code.
11221	* @param rcStrict Current EM status code.
11222	* @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
11223	* with @a rcStrict.
11224	* @param iMemMap The memory mapping index. For error reporting only.
11225	* @param pVCpu The cross context virtual CPU structure of the calling
11226	* thread, for error reporting only.
11227	*/
11228	DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu)
11229	{
11230	/* Simple. */
11231	if (RT_LIKELY(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RAW_TO_R3))
11232	return rcStrictCommit;
11233
11234	if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
11235	return rcStrict;
11236
11237	/* EM scheduling status codes. */
11238	if (RT_LIKELY( rcStrict >= VINF_EM_FIRST
11239	&& rcStrict <= VINF_EM_LAST))
11240	{
11241	if (RT_LIKELY( rcStrictCommit >= VINF_EM_FIRST
11242	&& rcStrictCommit <= VINF_EM_LAST))
11243	return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
11244	}
11245
11246	/* Unlikely */
11247	return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);
11248	}
11249
11250
11251	/**
11252	* Called by force-flag handling code when VMCPU_FF_IEM is set.
11253	*
11254	* @returns Merge between @a rcStrict and what the commit operation returned.
11255	* @param pVM The cross context VM structure.
11256	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11257	* @param rcStrict The status code returned by ring-0 or raw-mode.
11258	*/
11259	VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
11260	{
11261	/*
11262	* Reset the pending commit.
11263	*/
11264	AssertMsg( (pVCpu->iem.s.aMemMappings[0].fAccess \| pVCpu->iem.s.aMemMappings[1].fAccess \| pVCpu->iem.s.aMemMappings[2].fAccess)
11265	& (IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND),
11266	("%#x %#x %#x\n",
11267	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
11268	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);
11269
11270	/*
11271	* Commit the pending bounce buffers (usually just one).
11272	*/
11273	unsigned cBufs = 0;
11274	unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
11275	while (iMemMap-- > 0)
11276	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND))
11277	{
11278	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
11279	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
11280	Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);
11281
11282	uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
11283	uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
11284	uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
11285
11286	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)
11287	{
11288	VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,
11289	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
11290	pbBuf,
11291	cbFirst,
11292	PGMACCESSORIGIN_IEM);
11293	rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);
11294	Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",
11295	iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
11296	VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));
11297	}
11298
11299	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)
11300	{
11301	VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,
11302	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
11303	pbBuf + cbFirst,
11304	cbSecond,
11305	PGMACCESSORIGIN_IEM);
11306	rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);
11307	Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",
11308	iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,
11309	VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));
11310	}
11311	cBufs++;
11312	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
11313	}
11314
11315	AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,
11316	("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,
11317	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
11318	pVCpu->iem.s.cActiveMappings = 0;
11319	return rcStrict;
11320	}
11321
11322	#endif /* IN_RING3 */
11323

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 96622

Download in other formats: