IEMAllCImpl.cpp@ 97441

Last change on this file since 97441 was 97407, checked in by vboxsync, 2 years ago
VMM/IEM,CPUM: Partial single stepping support in the interpreter. [scm fix] bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 348.8 KB

Line
1	/* $Id: IEMAllCImpl.cpp 97407 2022-11-05 12:45:24Z vboxsync $ */
2	/** @file
3	* IEM - Instruction Implementation in C/C++ (code include).
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IEM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <VBox/vmm/iem.h>
35	#include <VBox/vmm/cpum.h>
36	#include <VBox/vmm/apic.h>
37	#include <VBox/vmm/pdm.h>
38	#include <VBox/vmm/pgm.h>
39	#include <VBox/vmm/iom.h>
40	#include <VBox/vmm/em.h>
41	#include <VBox/vmm/hm.h>
42	#include <VBox/vmm/nem.h>
43	#include <VBox/vmm/gim.h>
44	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
45	# include <VBox/vmm/em.h>
46	# include <VBox/vmm/hm_svm.h>
47	#endif
48	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
49	# include <VBox/vmm/hmvmxinline.h>
50	#endif
51	#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
52	# include <VBox/vmm/cpuidcall.h>
53	#endif
54	#include <VBox/vmm/tm.h>
55	#include <VBox/vmm/dbgf.h>
56	#include <VBox/vmm/dbgftrace.h>
57	#include "IEMInternal.h"
58	#include <VBox/vmm/vmcc.h>
59	#include <VBox/log.h>
60	#include <VBox/err.h>
61	#include <VBox/param.h>
62	#include <VBox/dis.h>
63	#include <VBox/disopcode.h>
64	#include <iprt/asm-math.h>
65	#include <iprt/assert.h>
66	#include <iprt/string.h>
67	#include <iprt/x86.h>
68
69	#include "IEMInline.h"
70
71
72	/** @name Misc Helpers
73	* @{
74	*/
75
76
77	/**
78	* Worker function for iemHlpCheckPortIOPermission, don't call directly.
79	*
80	* @returns Strict VBox status code.
81	*
82	* @param pVCpu The cross context virtual CPU structure of the calling thread.
83	* @param u16Port The port number.
84	* @param cbOperand The operand size.
85	*/
86	static VBOXSTRICTRC iemHlpCheckPortIOPermissionBitmap(PVMCPUCC pVCpu, uint16_t u16Port, uint8_t cbOperand)
87	{
88	/* The TSS bits we're interested in are the same on 386 and AMD64. */
89	AssertCompile(AMD64_SEL_TYPE_SYS_TSS_BUSY == X86_SEL_TYPE_SYS_386_TSS_BUSY);
90	AssertCompile(AMD64_SEL_TYPE_SYS_TSS_AVAIL == X86_SEL_TYPE_SYS_386_TSS_AVAIL);
91	AssertCompileMembersAtSameOffset(X86TSS32, offIoBitmap, X86TSS64, offIoBitmap);
92	AssertCompile(sizeof(X86TSS32) == sizeof(X86TSS64));
93
94	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
95
96	/*
97	* Check the TSS type, 16-bit TSSes doesn't have any I/O permission bitmap.
98	*/
99	Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType);
100	if (RT_UNLIKELY( pVCpu->cpum.GstCtx.tr.Attr.n.u4Type != AMD64_SEL_TYPE_SYS_TSS_BUSY
101	&& pVCpu->cpum.GstCtx.tr.Attr.n.u4Type != AMD64_SEL_TYPE_SYS_TSS_AVAIL))
102	{
103	Log(("iemHlpCheckPortIOPermissionBitmap: Port=%#x cb=%d - TSS type %#x (attr=%#x) has no I/O bitmap -> #GP(0)\n",
104	u16Port, cbOperand, pVCpu->cpum.GstCtx.tr.Attr.n.u4Type, pVCpu->cpum.GstCtx.tr.Attr.u));
105	return iemRaiseGeneralProtectionFault0(pVCpu);
106	}
107
108	/*
109	* Read the bitmap offset (may #PF).
110	*/
111	uint16_t offBitmap;
112	VBOXSTRICTRC rcStrict = iemMemFetchSysU16(pVCpu, &offBitmap, UINT8_MAX,
113	pVCpu->cpum.GstCtx.tr.u64Base + RT_UOFFSETOF(X86TSS64, offIoBitmap));
114	if (rcStrict != VINF_SUCCESS)
115	{
116	Log(("iemHlpCheckPortIOPermissionBitmap: Error reading offIoBitmap (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
117	return rcStrict;
118	}
119
120	/*
121	* The bit range from u16Port to (u16Port + cbOperand - 1), however intel
122	* describes the CPU actually reading two bytes regardless of whether the
123	* bit range crosses a byte boundrary. Thus the + 1 in the test below.
124	*/
125	uint32_t offFirstBit = (uint32_t)u16Port / 8 + offBitmap;
126	/** @todo check if real CPUs ensures that offBitmap has a minimum value of
127	* for instance sizeof(X86TSS32). */
128	if (offFirstBit + 1 > pVCpu->cpum.GstCtx.tr.u32Limit) /* the limit is inclusive */
129	{
130	Log(("iemHlpCheckPortIOPermissionBitmap: offFirstBit=%#x + 1 is beyond u32Limit=%#x -> #GP(0)\n",
131	offFirstBit, pVCpu->cpum.GstCtx.tr.u32Limit));
132	return iemRaiseGeneralProtectionFault0(pVCpu);
133	}
134
135	/*
136	* Read the necessary bits.
137	*/
138	/** @todo Test the assertion in the intel manual that the CPU reads two
139	* bytes. The question is how this works wrt to \#PF and \#GP on the
140	* 2nd byte when it's not required. */
141	uint16_t bmBytes = UINT16_MAX;
142	rcStrict = iemMemFetchSysU16(pVCpu, &bmBytes, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + offFirstBit);
143	if (rcStrict != VINF_SUCCESS)
144	{
145	Log(("iemHlpCheckPortIOPermissionBitmap: Error reading I/O bitmap @%#x (%Rrc)\n", offFirstBit, VBOXSTRICTRC_VAL(rcStrict)));
146	return rcStrict;
147	}
148
149	/*
150	* Perform the check.
151	*/
152	uint16_t fPortMask = (1 << cbOperand) - 1;
153	bmBytes >>= (u16Port & 7);
154	if (bmBytes & fPortMask)
155	{
156	Log(("iemHlpCheckPortIOPermissionBitmap: u16Port=%#x LB %u - access denied (bm=%#x mask=%#x) -> #GP(0)\n",
157	u16Port, cbOperand, bmBytes, fPortMask));
158	return iemRaiseGeneralProtectionFault0(pVCpu);
159	}
160
161	return VINF_SUCCESS;
162	}
163
164
165	/**
166	* Checks if we are allowed to access the given I/O port, raising the
167	* appropriate exceptions if we aren't (or if the I/O bitmap is not
168	* accessible).
169	*
170	* @returns Strict VBox status code.
171	*
172	* @param pVCpu The cross context virtual CPU structure of the calling thread.
173	* @param u16Port The port number.
174	* @param cbOperand The operand size.
175	*/
176	DECLINLINE(VBOXSTRICTRC) iemHlpCheckPortIOPermission(PVMCPUCC pVCpu, uint16_t u16Port, uint8_t cbOperand)
177	{
178	X86EFLAGS Efl;
179	Efl.u = IEMMISC_GET_EFL(pVCpu);
180	if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
181	&& ( pVCpu->iem.s.uCpl > Efl.Bits.u2IOPL
182	\|\| Efl.Bits.u1VM) )
183	return iemHlpCheckPortIOPermissionBitmap(pVCpu, u16Port, cbOperand);
184	return VINF_SUCCESS;
185	}
186
187
188	#if 0
189	/**
190	* Calculates the parity bit.
191	*
192	* @returns true if the bit is set, false if not.
193	* @param u8Result The least significant byte of the result.
194	*/
195	static bool iemHlpCalcParityFlag(uint8_t u8Result)
196	{
197	/*
198	* Parity is set if the number of bits in the least significant byte of
199	* the result is even.
200	*/
201	uint8_t cBits;
202	cBits = u8Result & 1; /* 0 */
203	u8Result >>= 1;
204	cBits += u8Result & 1;
205	u8Result >>= 1;
206	cBits += u8Result & 1;
207	u8Result >>= 1;
208	cBits += u8Result & 1;
209	u8Result >>= 1;
210	cBits += u8Result & 1; /* 4 */
211	u8Result >>= 1;
212	cBits += u8Result & 1;
213	u8Result >>= 1;
214	cBits += u8Result & 1;
215	u8Result >>= 1;
216	cBits += u8Result & 1;
217	return !(cBits & 1);
218	}
219	#endif /* not used */
220
221
222	/**
223	* Updates the specified flags according to a 8-bit result.
224	*
225	* @param pVCpu The cross context virtual CPU structure of the calling thread.
226	* @param u8Result The result to set the flags according to.
227	* @param fToUpdate The flags to update.
228	* @param fUndefined The flags that are specified as undefined.
229	*/
230	static void iemHlpUpdateArithEFlagsU8(PVMCPUCC pVCpu, uint8_t u8Result, uint32_t fToUpdate, uint32_t fUndefined)
231	{
232	uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
233	iemAImpl_test_u8(&u8Result, u8Result, &fEFlags);
234	pVCpu->cpum.GstCtx.eflags.u &= ~(fToUpdate \| fUndefined);
235	pVCpu->cpum.GstCtx.eflags.u \|= (fToUpdate \| fUndefined) & fEFlags;
236	}
237
238
239	/**
240	* Updates the specified flags according to a 16-bit result.
241	*
242	* @param pVCpu The cross context virtual CPU structure of the calling thread.
243	* @param u16Result The result to set the flags according to.
244	* @param fToUpdate The flags to update.
245	* @param fUndefined The flags that are specified as undefined.
246	*/
247	static void iemHlpUpdateArithEFlagsU16(PVMCPUCC pVCpu, uint16_t u16Result, uint32_t fToUpdate, uint32_t fUndefined)
248	{
249	uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
250	iemAImpl_test_u16(&u16Result, u16Result, &fEFlags);
251	pVCpu->cpum.GstCtx.eflags.u &= ~(fToUpdate \| fUndefined);
252	pVCpu->cpum.GstCtx.eflags.u \|= (fToUpdate \| fUndefined) & fEFlags;
253	}
254
255
256	/**
257	* Helper used by iret.
258	*
259	* @param pVCpu The cross context virtual CPU structure of the calling thread.
260	* @param uCpl The new CPL.
261	* @param pSReg Pointer to the segment register.
262	*/
263	static void iemHlpAdjustSelectorForNewCpl(PVMCPUCC pVCpu, uint8_t uCpl, PCPUMSELREG pSReg)
264	{
265	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
266	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
267
268	if ( uCpl > pSReg->Attr.n.u2Dpl
269	&& pSReg->Attr.n.u1DescType /* code or data, not system */
270	&& (pSReg->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
271	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF)) /* not conforming code */
272	iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, 0);
273	}
274
275
276	/**
277	* Indicates that we have modified the FPU state.
278	*
279	* @param pVCpu The cross context virtual CPU structure of the calling thread.
280	*/
281	DECLINLINE(void) iemHlpUsedFpu(PVMCPUCC pVCpu)
282	{
283	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
284	}
285
286	/** @} */
287
288	/** @name C Implementations
289	* @{
290	*/
291
292	/**
293	* Implements a 16-bit popa.
294	*/
295	IEM_CIMPL_DEF_0(iemCImpl_popa_16)
296	{
297	RTGCPTR GCPtrStart = iemRegGetEffRsp(pVCpu);
298	RTGCPTR GCPtrLast = GCPtrStart + 15;
299	VBOXSTRICTRC rcStrict;
300
301	/*
302	* The docs are a bit hard to comprehend here, but it looks like we wrap
303	* around in real mode as long as none of the individual "popa" crosses the
304	* end of the stack segment. In protected mode we check the whole access
305	* in one go. For efficiency, only do the word-by-word thing if we're in
306	* danger of wrapping around.
307	*/
308	/** @todo do popa boundary / wrap-around checks. */
309	if (RT_UNLIKELY( IEM_IS_REAL_OR_V86_MODE(pVCpu)
310	&& (pVCpu->cpum.GstCtx.cs.u32Limit < GCPtrLast)) ) /* ASSUMES 64-bit RTGCPTR */
311	{
312	/* word-by-word */
313	RTUINT64U TmpRsp;
314	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
315	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.di, &TmpRsp);
316	if (rcStrict == VINF_SUCCESS)
317	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.si, &TmpRsp);
318	if (rcStrict == VINF_SUCCESS)
319	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.bp, &TmpRsp);
320	if (rcStrict == VINF_SUCCESS)
321	{
322	iemRegAddToRspEx(pVCpu, &TmpRsp, 2); /* sp */
323	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.bx, &TmpRsp);
324	}
325	if (rcStrict == VINF_SUCCESS)
326	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.dx, &TmpRsp);
327	if (rcStrict == VINF_SUCCESS)
328	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.cx, &TmpRsp);
329	if (rcStrict == VINF_SUCCESS)
330	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.ax, &TmpRsp);
331	if (rcStrict == VINF_SUCCESS)
332	{
333	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
334	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
335	}
336	}
337	else
338	{
339	uint16_t const *pa16Mem = NULL;
340	rcStrict = iemMemMap(pVCpu, (void *)&pa16Mem, 16, X86_SREG_SS, GCPtrStart, IEM_ACCESS_STACK_R, sizeof(pa16Mem) - 1);
341	if (rcStrict == VINF_SUCCESS)
342	{
343	pVCpu->cpum.GstCtx.di = pa16Mem[7 - X86_GREG_xDI];
344	pVCpu->cpum.GstCtx.si = pa16Mem[7 - X86_GREG_xSI];
345	pVCpu->cpum.GstCtx.bp = pa16Mem[7 - X86_GREG_xBP];
346	/* skip sp */
347	pVCpu->cpum.GstCtx.bx = pa16Mem[7 - X86_GREG_xBX];
348	pVCpu->cpum.GstCtx.dx = pa16Mem[7 - X86_GREG_xDX];
349	pVCpu->cpum.GstCtx.cx = pa16Mem[7 - X86_GREG_xCX];
350	pVCpu->cpum.GstCtx.ax = pa16Mem[7 - X86_GREG_xAX];
351	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa16Mem, IEM_ACCESS_STACK_R);
352	if (rcStrict == VINF_SUCCESS)
353	{
354	iemRegAddToRsp(pVCpu, 16);
355	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
356	}
357	}
358	}
359	return rcStrict;
360	}
361
362
363	/**
364	* Implements a 32-bit popa.
365	*/
366	IEM_CIMPL_DEF_0(iemCImpl_popa_32)
367	{
368	RTGCPTR GCPtrStart = iemRegGetEffRsp(pVCpu);
369	RTGCPTR GCPtrLast = GCPtrStart + 31;
370	VBOXSTRICTRC rcStrict;
371
372	/*
373	* The docs are a bit hard to comprehend here, but it looks like we wrap
374	* around in real mode as long as none of the individual "popa" crosses the
375	* end of the stack segment. In protected mode we check the whole access
376	* in one go. For efficiency, only do the word-by-word thing if we're in
377	* danger of wrapping around.
378	*/
379	/** @todo do popa boundary / wrap-around checks. */
380	if (RT_UNLIKELY( IEM_IS_REAL_OR_V86_MODE(pVCpu)
381	&& (pVCpu->cpum.GstCtx.cs.u32Limit < GCPtrLast)) ) /* ASSUMES 64-bit RTGCPTR */
382	{
383	/* word-by-word */
384	RTUINT64U TmpRsp;
385	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
386	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.edi, &TmpRsp);
387	if (rcStrict == VINF_SUCCESS)
388	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.esi, &TmpRsp);
389	if (rcStrict == VINF_SUCCESS)
390	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ebp, &TmpRsp);
391	if (rcStrict == VINF_SUCCESS)
392	{
393	iemRegAddToRspEx(pVCpu, &TmpRsp, 2); /* sp */
394	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ebx, &TmpRsp);
395	}
396	if (rcStrict == VINF_SUCCESS)
397	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.edx, &TmpRsp);
398	if (rcStrict == VINF_SUCCESS)
399	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ecx, &TmpRsp);
400	if (rcStrict == VINF_SUCCESS)
401	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.eax, &TmpRsp);
402	if (rcStrict == VINF_SUCCESS)
403	{
404	#if 1 /** @todo what actually happens with the high bits when we're in 16-bit mode? */
405	pVCpu->cpum.GstCtx.rdi &= UINT32_MAX;
406	pVCpu->cpum.GstCtx.rsi &= UINT32_MAX;
407	pVCpu->cpum.GstCtx.rbp &= UINT32_MAX;
408	pVCpu->cpum.GstCtx.rbx &= UINT32_MAX;
409	pVCpu->cpum.GstCtx.rdx &= UINT32_MAX;
410	pVCpu->cpum.GstCtx.rcx &= UINT32_MAX;
411	pVCpu->cpum.GstCtx.rax &= UINT32_MAX;
412	#endif
413	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
414	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
415	}
416	}
417	else
418	{
419	uint32_t const *pa32Mem;
420	rcStrict = iemMemMap(pVCpu, (void *)&pa32Mem, 32, X86_SREG_SS, GCPtrStart, IEM_ACCESS_STACK_R, sizeof(pa32Mem) - 1);
421	if (rcStrict == VINF_SUCCESS)
422	{
423	pVCpu->cpum.GstCtx.rdi = pa32Mem[7 - X86_GREG_xDI];
424	pVCpu->cpum.GstCtx.rsi = pa32Mem[7 - X86_GREG_xSI];
425	pVCpu->cpum.GstCtx.rbp = pa32Mem[7 - X86_GREG_xBP];
426	/* skip esp */
427	pVCpu->cpum.GstCtx.rbx = pa32Mem[7 - X86_GREG_xBX];
428	pVCpu->cpum.GstCtx.rdx = pa32Mem[7 - X86_GREG_xDX];
429	pVCpu->cpum.GstCtx.rcx = pa32Mem[7 - X86_GREG_xCX];
430	pVCpu->cpum.GstCtx.rax = pa32Mem[7 - X86_GREG_xAX];
431	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa32Mem, IEM_ACCESS_STACK_R);
432	if (rcStrict == VINF_SUCCESS)
433	{
434	iemRegAddToRsp(pVCpu, 32);
435	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
436	}
437	}
438	}
439	return rcStrict;
440	}
441
442
443	/**
444	* Implements a 16-bit pusha.
445	*/
446	IEM_CIMPL_DEF_0(iemCImpl_pusha_16)
447	{
448	RTGCPTR GCPtrTop = iemRegGetEffRsp(pVCpu);
449	RTGCPTR GCPtrBottom = GCPtrTop - 15;
450	VBOXSTRICTRC rcStrict;
451
452	/*
453	* The docs are a bit hard to comprehend here, but it looks like we wrap
454	* around in real mode as long as none of the individual "pushd" crosses the
455	* end of the stack segment. In protected mode we check the whole access
456	* in one go. For efficiency, only do the word-by-word thing if we're in
457	* danger of wrapping around.
458	*/
459	/** @todo do pusha boundary / wrap-around checks. */
460	if (RT_UNLIKELY( GCPtrBottom > GCPtrTop
461	&& IEM_IS_REAL_OR_V86_MODE(pVCpu) ) )
462	{
463	/* word-by-word */
464	RTUINT64U TmpRsp;
465	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
466	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.ax, &TmpRsp);
467	if (rcStrict == VINF_SUCCESS)
468	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.cx, &TmpRsp);
469	if (rcStrict == VINF_SUCCESS)
470	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.dx, &TmpRsp);
471	if (rcStrict == VINF_SUCCESS)
472	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.bx, &TmpRsp);
473	if (rcStrict == VINF_SUCCESS)
474	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.sp, &TmpRsp);
475	if (rcStrict == VINF_SUCCESS)
476	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.bp, &TmpRsp);
477	if (rcStrict == VINF_SUCCESS)
478	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.si, &TmpRsp);
479	if (rcStrict == VINF_SUCCESS)
480	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.di, &TmpRsp);
481	if (rcStrict == VINF_SUCCESS)
482	{
483	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
484	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
485	}
486	}
487	else
488	{
489	GCPtrBottom--;
490	uint16_t *pa16Mem = NULL;
491	rcStrict = iemMemMap(pVCpu, (void *)&pa16Mem, 16, X86_SREG_SS, GCPtrBottom, IEM_ACCESS_STACK_W, sizeof(pa16Mem) - 1);
492	if (rcStrict == VINF_SUCCESS)
493	{
494	pa16Mem[7 - X86_GREG_xDI] = pVCpu->cpum.GstCtx.di;
495	pa16Mem[7 - X86_GREG_xSI] = pVCpu->cpum.GstCtx.si;
496	pa16Mem[7 - X86_GREG_xBP] = pVCpu->cpum.GstCtx.bp;
497	pa16Mem[7 - X86_GREG_xSP] = pVCpu->cpum.GstCtx.sp;
498	pa16Mem[7 - X86_GREG_xBX] = pVCpu->cpum.GstCtx.bx;
499	pa16Mem[7 - X86_GREG_xDX] = pVCpu->cpum.GstCtx.dx;
500	pa16Mem[7 - X86_GREG_xCX] = pVCpu->cpum.GstCtx.cx;
501	pa16Mem[7 - X86_GREG_xAX] = pVCpu->cpum.GstCtx.ax;
502	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa16Mem, IEM_ACCESS_STACK_W);
503	if (rcStrict == VINF_SUCCESS)
504	{
505	iemRegSubFromRsp(pVCpu, 16);
506	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
507	}
508	}
509	}
510	return rcStrict;
511	}
512
513
514	/**
515	* Implements a 32-bit pusha.
516	*/
517	IEM_CIMPL_DEF_0(iemCImpl_pusha_32)
518	{
519	RTGCPTR GCPtrTop = iemRegGetEffRsp(pVCpu);
520	RTGCPTR GCPtrBottom = GCPtrTop - 31;
521	VBOXSTRICTRC rcStrict;
522
523	/*
524	* The docs are a bit hard to comprehend here, but it looks like we wrap
525	* around in real mode as long as none of the individual "pusha" crosses the
526	* end of the stack segment. In protected mode we check the whole access
527	* in one go. For efficiency, only do the word-by-word thing if we're in
528	* danger of wrapping around.
529	*/
530	/** @todo do pusha boundary / wrap-around checks. */
531	if (RT_UNLIKELY( GCPtrBottom > GCPtrTop
532	&& IEM_IS_REAL_OR_V86_MODE(pVCpu) ) )
533	{
534	/* word-by-word */
535	RTUINT64U TmpRsp;
536	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
537	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.eax, &TmpRsp);
538	if (rcStrict == VINF_SUCCESS)
539	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ecx, &TmpRsp);
540	if (rcStrict == VINF_SUCCESS)
541	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.edx, &TmpRsp);
542	if (rcStrict == VINF_SUCCESS)
543	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ebx, &TmpRsp);
544	if (rcStrict == VINF_SUCCESS)
545	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.esp, &TmpRsp);
546	if (rcStrict == VINF_SUCCESS)
547	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ebp, &TmpRsp);
548	if (rcStrict == VINF_SUCCESS)
549	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.esi, &TmpRsp);
550	if (rcStrict == VINF_SUCCESS)
551	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.edi, &TmpRsp);
552	if (rcStrict == VINF_SUCCESS)
553	{
554	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
555	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
556	}
557	}
558	else
559	{
560	GCPtrBottom--;
561	uint32_t *pa32Mem;
562	rcStrict = iemMemMap(pVCpu, (void *)&pa32Mem, 32, X86_SREG_SS, GCPtrBottom, IEM_ACCESS_STACK_W, sizeof(pa32Mem) - 1);
563	if (rcStrict == VINF_SUCCESS)
564	{
565	pa32Mem[7 - X86_GREG_xDI] = pVCpu->cpum.GstCtx.edi;
566	pa32Mem[7 - X86_GREG_xSI] = pVCpu->cpum.GstCtx.esi;
567	pa32Mem[7 - X86_GREG_xBP] = pVCpu->cpum.GstCtx.ebp;
568	pa32Mem[7 - X86_GREG_xSP] = pVCpu->cpum.GstCtx.esp;
569	pa32Mem[7 - X86_GREG_xBX] = pVCpu->cpum.GstCtx.ebx;
570	pa32Mem[7 - X86_GREG_xDX] = pVCpu->cpum.GstCtx.edx;
571	pa32Mem[7 - X86_GREG_xCX] = pVCpu->cpum.GstCtx.ecx;
572	pa32Mem[7 - X86_GREG_xAX] = pVCpu->cpum.GstCtx.eax;
573	rcStrict = iemMemCommitAndUnmap(pVCpu, pa32Mem, IEM_ACCESS_STACK_W);
574	if (rcStrict == VINF_SUCCESS)
575	{
576	iemRegSubFromRsp(pVCpu, 32);
577	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
578	}
579	}
580	}
581	return rcStrict;
582	}
583
584
585	/**
586	* Implements pushf.
587	*
588	*
589	* @param enmEffOpSize The effective operand size.
590	*/
591	IEM_CIMPL_DEF_1(iemCImpl_pushf, IEMMODE, enmEffOpSize)
592	{
593	VBOXSTRICTRC rcStrict;
594
595	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_PUSHF))
596	{
597	Log2(("pushf: Guest intercept -> #VMEXIT\n"));
598	IEM_SVM_UPDATE_NRIP(pVCpu);
599	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_PUSHF, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
600	}
601
602	/*
603	* If we're in V8086 mode some care is required (which is why we're in
604	* doing this in a C implementation).
605	*/
606	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
607	if ( (fEfl & X86_EFL_VM)
608	&& X86_EFL_GET_IOPL(fEfl) != 3 )
609	{
610	Assert(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE);
611	if ( enmEffOpSize != IEMMODE_16BIT
612	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME))
613	return iemRaiseGeneralProtectionFault0(pVCpu);
614	fEfl &= ~X86_EFL_IF; /* (RF and VM are out of range) */
615	fEfl \|= (fEfl & X86_EFL_VIF) >> (19 - 9);
616	rcStrict = iemMemStackPushU16(pVCpu, (uint16_t)fEfl);
617	}
618	else
619	{
620
621	/*
622	* Ok, clear RF and VM, adjust for ancient CPUs, and push the flags.
623	*/
624	fEfl &= ~(X86_EFL_RF \| X86_EFL_VM);
625
626	switch (enmEffOpSize)
627	{
628	case IEMMODE_16BIT:
629	AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
630	if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_186)
631	fEfl \|= UINT16_C(0xf000);
632	rcStrict = iemMemStackPushU16(pVCpu, (uint16_t)fEfl);
633	break;
634	case IEMMODE_32BIT:
635	rcStrict = iemMemStackPushU32(pVCpu, fEfl);
636	break;
637	case IEMMODE_64BIT:
638	rcStrict = iemMemStackPushU64(pVCpu, fEfl);
639	break;
640	IEM_NOT_REACHED_DEFAULT_CASE_RET();
641	}
642	}
643
644	if (rcStrict == VINF_SUCCESS)
645	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
646	return rcStrict;
647	}
648
649
650	/**
651	* Implements popf.
652	*
653	* @param enmEffOpSize The effective operand size.
654	*/
655	IEM_CIMPL_DEF_1(iemCImpl_popf, IEMMODE, enmEffOpSize)
656	{
657	uint32_t const fEflOld = IEMMISC_GET_EFL(pVCpu);
658	VBOXSTRICTRC rcStrict;
659	uint32_t fEflNew;
660
661	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_POPF))
662	{
663	Log2(("popf: Guest intercept -> #VMEXIT\n"));
664	IEM_SVM_UPDATE_NRIP(pVCpu);
665	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_POPF, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
666	}
667
668	/*
669	* V8086 is special as usual.
670	*/
671	if (fEflOld & X86_EFL_VM)
672	{
673	/*
674	* Almost anything goes if IOPL is 3.
675	*/
676	if (X86_EFL_GET_IOPL(fEflOld) == 3)
677	{
678	switch (enmEffOpSize)
679	{
680	case IEMMODE_16BIT:
681	{
682	uint16_t u16Value;
683	rcStrict = iemMemStackPopU16(pVCpu, &u16Value);
684	if (rcStrict != VINF_SUCCESS)
685	return rcStrict;
686	fEflNew = u16Value \| (fEflOld & UINT32_C(0xffff0000));
687	break;
688	}
689	case IEMMODE_32BIT:
690	rcStrict = iemMemStackPopU32(pVCpu, &fEflNew);
691	if (rcStrict != VINF_SUCCESS)
692	return rcStrict;
693	break;
694	IEM_NOT_REACHED_DEFAULT_CASE_RET();
695	}
696
697	const uint32_t fPopfBits = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386
698	? X86_EFL_POPF_BITS : X86_EFL_POPF_BITS_386;
699	fEflNew &= fPopfBits & ~(X86_EFL_IOPL);
700	fEflNew \|= ~(fPopfBits & ~(X86_EFL_IOPL)) & fEflOld;
701	}
702	/*
703	* Interrupt flag virtualization with CR4.VME=1.
704	*/
705	else if ( enmEffOpSize == IEMMODE_16BIT
706	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) )
707	{
708	uint16_t u16Value;
709	RTUINT64U TmpRsp;
710	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
711	rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Value, &TmpRsp);
712	if (rcStrict != VINF_SUCCESS)
713	return rcStrict;
714
715	/** @todo Is the popf VME \#GP(0) delivered after updating RSP+RIP
716	* or before? */
717	if ( ( (u16Value & X86_EFL_IF)
718	&& (fEflOld & X86_EFL_VIP))
719	\|\| (u16Value & X86_EFL_TF) )
720	return iemRaiseGeneralProtectionFault0(pVCpu);
721
722	fEflNew = u16Value \| (fEflOld & UINT32_C(0xffff0000) & ~X86_EFL_VIF);
723	fEflNew \|= (fEflNew & X86_EFL_IF) << (19 - 9);
724	fEflNew &= X86_EFL_POPF_BITS & ~(X86_EFL_IOPL \| X86_EFL_IF);
725	fEflNew \|= ~(X86_EFL_POPF_BITS & ~(X86_EFL_IOPL \| X86_EFL_IF)) & fEflOld;
726
727	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
728	}
729	else
730	return iemRaiseGeneralProtectionFault0(pVCpu);
731
732	}
733	/*
734	* Not in V8086 mode.
735	*/
736	else
737	{
738	/* Pop the flags. */
739	switch (enmEffOpSize)
740	{
741	case IEMMODE_16BIT:
742	{
743	uint16_t u16Value;
744	rcStrict = iemMemStackPopU16(pVCpu, &u16Value);
745	if (rcStrict != VINF_SUCCESS)
746	return rcStrict;
747	fEflNew = u16Value \| (fEflOld & UINT32_C(0xffff0000));
748
749	/*
750	* Ancient CPU adjustments:
751	* - 8086, 80186, V20/30:
752	* Fixed bits 15:12 bits are not kept correctly internally, mostly for
753	* practical reasons (masking below). We add them when pushing flags.
754	* - 80286:
755	* The NT and IOPL flags cannot be popped from real mode and are
756	* therefore always zero (since a 286 can never exit from PM and
757	* their initial value is zero). This changed on a 386 and can
758	* therefore be used to detect 286 or 386 CPU in real mode.
759	*/
760	if ( IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_286
761	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
762	fEflNew &= ~(X86_EFL_NT \| X86_EFL_IOPL);
763	break;
764	}
765	case IEMMODE_32BIT:
766	rcStrict = iemMemStackPopU32(pVCpu, &fEflNew);
767	if (rcStrict != VINF_SUCCESS)
768	return rcStrict;
769	break;
770	case IEMMODE_64BIT:
771	{
772	uint64_t u64Value;
773	rcStrict = iemMemStackPopU64(pVCpu, &u64Value);
774	if (rcStrict != VINF_SUCCESS)
775	return rcStrict;
776	fEflNew = u64Value; /** @todo testcase: Check exactly what happens if high bits are set. */
777	break;
778	}
779	IEM_NOT_REACHED_DEFAULT_CASE_RET();
780	}
781
782	/* Merge them with the current flags. */
783	const uint32_t fPopfBits = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386
784	? X86_EFL_POPF_BITS : X86_EFL_POPF_BITS_386;
785	if ( (fEflNew & (X86_EFL_IOPL \| X86_EFL_IF)) == (fEflOld & (X86_EFL_IOPL \| X86_EFL_IF))
786	\|\| pVCpu->iem.s.uCpl == 0)
787	{
788	fEflNew &= fPopfBits;
789	fEflNew \|= ~fPopfBits & fEflOld;
790	}
791	else if (pVCpu->iem.s.uCpl <= X86_EFL_GET_IOPL(fEflOld))
792	{
793	fEflNew &= fPopfBits & ~(X86_EFL_IOPL);
794	fEflNew \|= ~(fPopfBits & ~(X86_EFL_IOPL)) & fEflOld;
795	}
796	else
797	{
798	fEflNew &= fPopfBits & ~(X86_EFL_IOPL \| X86_EFL_IF);
799	fEflNew \|= ~(fPopfBits & ~(X86_EFL_IOPL \| X86_EFL_IF)) & fEflOld;
800	}
801	}
802
803	/*
804	* Commit the flags.
805	*/
806	Assert(fEflNew & RT_BIT_32(1));
807	IEMMISC_SET_EFL(pVCpu, fEflNew);
808	return iemRegAddToRipAndFinishingClearingRfEx(pVCpu, cbInstr, fEflOld);
809	}
810
811
812	/**
813	* Implements an indirect call.
814	*
815	* @param uNewPC The new program counter (RIP) value (loaded from the
816	* operand).
817	*/
818	IEM_CIMPL_DEF_1(iemCImpl_call_16, uint16_t, uNewPC)
819	{
820	uint16_t uOldPC = pVCpu->cpum.GstCtx.ip + cbInstr;
821	if (uNewPC > pVCpu->cpum.GstCtx.cs.u32Limit)
822	return iemRaiseGeneralProtectionFault0(pVCpu);
823
824	VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldPC);
825	if (rcStrict != VINF_SUCCESS)
826	return rcStrict;
827
828	pVCpu->cpum.GstCtx.rip = uNewPC;
829	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
830
831	#ifndef IEM_WITH_CODE_TLB
832	/* Flush the prefetch buffer. */
833	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
834	#endif
835	return VINF_SUCCESS;
836	}
837
838
839	/**
840	* Implements a 16-bit relative call.
841	*
842	* @param offDisp The displacment offset.
843	*/
844	IEM_CIMPL_DEF_1(iemCImpl_call_rel_16, int16_t, offDisp)
845	{
846	uint16_t uOldPC = pVCpu->cpum.GstCtx.ip + cbInstr;
847	uint16_t uNewPC = uOldPC + offDisp;
848	if (uNewPC > pVCpu->cpum.GstCtx.cs.u32Limit)
849	return iemRaiseGeneralProtectionFault0(pVCpu);
850
851	VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldPC);
852	if (rcStrict != VINF_SUCCESS)
853	return rcStrict;
854
855	pVCpu->cpum.GstCtx.rip = uNewPC;
856	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
857
858	#ifndef IEM_WITH_CODE_TLB
859	/* Flush the prefetch buffer. */
860	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
861	#endif
862	return VINF_SUCCESS;
863	}
864
865
866	/**
867	* Implements a 32-bit indirect call.
868	*
869	* @param uNewPC The new program counter (RIP) value (loaded from the
870	* operand).
871	*/
872	IEM_CIMPL_DEF_1(iemCImpl_call_32, uint32_t, uNewPC)
873	{
874	uint32_t uOldPC = pVCpu->cpum.GstCtx.eip + cbInstr;
875	if (uNewPC > pVCpu->cpum.GstCtx.cs.u32Limit)
876	return iemRaiseGeneralProtectionFault0(pVCpu);
877
878	VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldPC);
879	if (rcStrict != VINF_SUCCESS)
880	return rcStrict;
881
882	pVCpu->cpum.GstCtx.rip = uNewPC;
883	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
884
885	#ifndef IEM_WITH_CODE_TLB
886	/* Flush the prefetch buffer. */
887	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
888	#endif
889	return VINF_SUCCESS;
890	}
891
892
893	/**
894	* Implements a 32-bit relative call.
895	*
896	* @param offDisp The displacment offset.
897	*/
898	IEM_CIMPL_DEF_1(iemCImpl_call_rel_32, int32_t, offDisp)
899	{
900	uint32_t uOldPC = pVCpu->cpum.GstCtx.eip + cbInstr;
901	uint32_t uNewPC = uOldPC + offDisp;
902	if (uNewPC > pVCpu->cpum.GstCtx.cs.u32Limit)
903	return iemRaiseGeneralProtectionFault0(pVCpu);
904
905	VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldPC);
906	if (rcStrict != VINF_SUCCESS)
907	return rcStrict;
908
909	pVCpu->cpum.GstCtx.rip = uNewPC;
910	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
911
912	#ifndef IEM_WITH_CODE_TLB
913	/* Flush the prefetch buffer. */
914	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
915	#endif
916	return VINF_SUCCESS;
917	}
918
919
920	/**
921	* Implements a 64-bit indirect call.
922	*
923	* @param uNewPC The new program counter (RIP) value (loaded from the
924	* operand).
925	*/
926	IEM_CIMPL_DEF_1(iemCImpl_call_64, uint64_t, uNewPC)
927	{
928	uint64_t uOldPC = pVCpu->cpum.GstCtx.rip + cbInstr;
929	if (!IEM_IS_CANONICAL(uNewPC))
930	return iemRaiseGeneralProtectionFault0(pVCpu);
931
932	VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldPC);
933	if (rcStrict != VINF_SUCCESS)
934	return rcStrict;
935
936	pVCpu->cpum.GstCtx.rip = uNewPC;
937	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
938
939	#ifndef IEM_WITH_CODE_TLB
940	/* Flush the prefetch buffer. */
941	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
942	#endif
943	return VINF_SUCCESS;
944	}
945
946
947	/**
948	* Implements a 64-bit relative call.
949	*
950	* @param offDisp The displacment offset.
951	*/
952	IEM_CIMPL_DEF_1(iemCImpl_call_rel_64, int64_t, offDisp)
953	{
954	uint64_t uOldPC = pVCpu->cpum.GstCtx.rip + cbInstr;
955	uint64_t uNewPC = uOldPC + offDisp;
956	if (!IEM_IS_CANONICAL(uNewPC))
957	return iemRaiseNotCanonical(pVCpu);
958
959	VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldPC);
960	if (rcStrict != VINF_SUCCESS)
961	return rcStrict;
962
963	pVCpu->cpum.GstCtx.rip = uNewPC;
964	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
965
966	#ifndef IEM_WITH_CODE_TLB
967	/* Flush the prefetch buffer. */
968	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
969	#endif
970
971	return VINF_SUCCESS;
972	}
973
974
975	/**
976	* Implements far jumps and calls thru task segments (TSS).
977	*
978	* @param uSel The selector.
979	* @param enmBranch The kind of branching we're performing.
980	* @param enmEffOpSize The effective operand size.
981	* @param pDesc The descriptor corresponding to @a uSel. The type is
982	* task gate.
983	*/
984	IEM_CIMPL_DEF_4(iemCImpl_BranchTaskSegment, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc)
985	{
986	#ifndef IEM_IMPLEMENTS_TASKSWITCH
987	IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
988	#else
989	Assert(enmBranch == IEMBRANCH_JUMP \|\| enmBranch == IEMBRANCH_CALL);
990	Assert( pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
991	\|\| pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL);
992	RT_NOREF_PV(enmEffOpSize);
993	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
994
995	if ( pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl
996	\|\| pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
997	{
998	Log(("BranchTaskSegment invalid priv. uSel=%04x TSS DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
999	pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL)));
1000	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
1001	}
1002
1003	/** @todo This is checked earlier for far jumps (see iemCImpl_FarJmp) but not
1004	* far calls (see iemCImpl_callf). Most likely in both cases it should be
1005	* checked here, need testcases. */
1006	if (!pDesc->Legacy.Gen.u1Present)
1007	{
1008	Log(("BranchTaskSegment TSS not present uSel=%04x -> #NP\n", uSel));
1009	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
1010	}
1011
1012	uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
1013	return iemTaskSwitch(pVCpu, enmBranch == IEMBRANCH_JUMP ? IEMTASKSWITCH_JUMP : IEMTASKSWITCH_CALL,
1014	uNextEip, 0 /* fFlags /, 0 / uErr /, 0 / uCr2 */, uSel, pDesc);
1015	#endif
1016	}
1017
1018
1019	/**
1020	* Implements far jumps and calls thru task gates.
1021	*
1022	* @param uSel The selector.
1023	* @param enmBranch The kind of branching we're performing.
1024	* @param enmEffOpSize The effective operand size.
1025	* @param pDesc The descriptor corresponding to @a uSel. The type is
1026	* task gate.
1027	*/
1028	IEM_CIMPL_DEF_4(iemCImpl_BranchTaskGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc)
1029	{
1030	#ifndef IEM_IMPLEMENTS_TASKSWITCH
1031	IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
1032	#else
1033	Assert(enmBranch == IEMBRANCH_JUMP \|\| enmBranch == IEMBRANCH_CALL);
1034	RT_NOREF_PV(enmEffOpSize);
1035	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
1036
1037	if ( pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl
1038	\|\| pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
1039	{
1040	Log(("BranchTaskGate invalid priv. uSel=%04x TSS DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
1041	pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL)));
1042	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
1043	}
1044
1045	/** @todo This is checked earlier for far jumps (see iemCImpl_FarJmp) but not
1046	* far calls (see iemCImpl_callf). Most likely in both cases it should be
1047	* checked here, need testcases. */
1048	if (!pDesc->Legacy.Gen.u1Present)
1049	{
1050	Log(("BranchTaskSegment segment not present uSel=%04x -> #NP\n", uSel));
1051	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
1052	}
1053
1054	/*
1055	* Fetch the new TSS descriptor from the GDT.
1056	*/
1057	RTSEL uSelTss = pDesc->Legacy.Gate.u16Sel;
1058	if (uSelTss & X86_SEL_LDT)
1059	{
1060	Log(("BranchTaskGate TSS is in LDT. uSel=%04x uSelTss=%04x -> #GP\n", uSel, uSelTss));
1061	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
1062	}
1063
1064	IEMSELDESC TssDesc;
1065	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &TssDesc, uSelTss, X86_XCPT_GP);
1066	if (rcStrict != VINF_SUCCESS)
1067	return rcStrict;
1068
1069	if (TssDesc.Legacy.Gate.u4Type & X86_SEL_TYPE_SYS_TSS_BUSY_MASK)
1070	{
1071	Log(("BranchTaskGate TSS is busy. uSel=%04x uSelTss=%04x DescType=%#x -> #GP\n", uSel, uSelTss,
1072	TssDesc.Legacy.Gate.u4Type));
1073	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
1074	}
1075
1076	if (!TssDesc.Legacy.Gate.u1Present)
1077	{
1078	Log(("BranchTaskGate TSS is not present. uSel=%04x uSelTss=%04x -> #NP\n", uSel, uSelTss));
1079	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSelTss & X86_SEL_MASK_OFF_RPL);
1080	}
1081
1082	uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
1083	return iemTaskSwitch(pVCpu, enmBranch == IEMBRANCH_JUMP ? IEMTASKSWITCH_JUMP : IEMTASKSWITCH_CALL,
1084	uNextEip, 0 /* fFlags /, 0 / uErr /, 0 / uCr2 */, uSelTss, &TssDesc);
1085	#endif
1086	}
1087
1088
1089	/**
1090	* Implements far jumps and calls thru call gates.
1091	*
1092	* @param uSel The selector.
1093	* @param enmBranch The kind of branching we're performing.
1094	* @param enmEffOpSize The effective operand size.
1095	* @param pDesc The descriptor corresponding to @a uSel. The type is
1096	* call gate.
1097	*/
1098	IEM_CIMPL_DEF_4(iemCImpl_BranchCallGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc)
1099	{
1100	#define IEM_IMPLEMENTS_CALLGATE
1101	#ifndef IEM_IMPLEMENTS_CALLGATE
1102	IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
1103	#else
1104	RT_NOREF_PV(enmEffOpSize);
1105	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
1106
1107	/* NB: Far jumps can only do intra-privilege transfers. Far calls support
1108	* inter-privilege calls and are much more complex.
1109	*
1110	* NB: 64-bit call gate has the same type as a 32-bit call gate! If
1111	* EFER.LMA=1, the gate must be 64-bit. Conversely if EFER.LMA=0, the gate
1112	* must be 16-bit or 32-bit.
1113	*/
1114	/** @todo effective operand size is probably irrelevant here, only the
1115	* call gate bitness matters??
1116	*/
1117	VBOXSTRICTRC rcStrict;
1118	RTPTRUNION uPtrRet;
1119	uint64_t uNewRsp;
1120	uint64_t uNewRip;
1121	uint64_t u64Base;
1122	uint32_t cbLimit;
1123	RTSEL uNewCS;
1124	IEMSELDESC DescCS;
1125
1126	AssertCompile(X86_SEL_TYPE_SYS_386_CALL_GATE == AMD64_SEL_TYPE_SYS_CALL_GATE);
1127	Assert(enmBranch == IEMBRANCH_JUMP \|\| enmBranch == IEMBRANCH_CALL);
1128	Assert( pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE
1129	\|\| pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE);
1130
1131	/* Determine the new instruction pointer from the gate descriptor. */
1132	uNewRip = pDesc->Legacy.Gate.u16OffsetLow
1133	\| ((uint32_t)pDesc->Legacy.Gate.u16OffsetHigh << 16)
1134	\| ((uint64_t)pDesc->Long.Gate.u32OffsetTop << 32);
1135
1136	/* Perform DPL checks on the gate descriptor. */
1137	if ( pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl
1138	\|\| pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
1139	{
1140	Log(("BranchCallGate invalid priv. uSel=%04x Gate DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
1141	pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL)));
1142	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1143	}
1144
1145	/** @todo does this catch NULL selectors, too? */
1146	if (!pDesc->Legacy.Gen.u1Present)
1147	{
1148	Log(("BranchCallGate Gate not present uSel=%04x -> #NP\n", uSel));
1149	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
1150	}
1151
1152	/*
1153	* Fetch the target CS descriptor from the GDT or LDT.
1154	*/
1155	uNewCS = pDesc->Legacy.Gate.u16Sel;
1156	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_GP);
1157	if (rcStrict != VINF_SUCCESS)
1158	return rcStrict;
1159
1160	/* Target CS must be a code selector. */
1161	if ( !DescCS.Legacy.Gen.u1DescType
1162	\|\| !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) )
1163	{
1164	Log(("BranchCallGate %04x:%08RX64 -> not a code selector (u1DescType=%u u4Type=%#x).\n",
1165	uNewCS, uNewRip, DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
1166	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
1167	}
1168
1169	/* Privilege checks on target CS. */
1170	if (enmBranch == IEMBRANCH_JUMP)
1171	{
1172	if (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
1173	{
1174	if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
1175	{
1176	Log(("BranchCallGate jump (conforming) bad DPL uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
1177	uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
1178	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
1179	}
1180	}
1181	else
1182	{
1183	if (DescCS.Legacy.Gen.u2Dpl != pVCpu->iem.s.uCpl)
1184	{
1185	Log(("BranchCallGate jump (non-conforming) bad DPL uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
1186	uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
1187	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
1188	}
1189	}
1190	}
1191	else
1192	{
1193	Assert(enmBranch == IEMBRANCH_CALL);
1194	if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
1195	{
1196	Log(("BranchCallGate call invalid priv. uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
1197	uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
1198	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
1199	}
1200	}
1201
1202	/* Additional long mode checks. */
1203	if (IEM_IS_LONG_MODE(pVCpu))
1204	{
1205	if (!DescCS.Legacy.Gen.u1Long)
1206	{
1207	Log(("BranchCallGate uNewCS %04x -> not a 64-bit code segment.\n", uNewCS));
1208	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
1209	}
1210
1211	/* L vs D. */
1212	if ( DescCS.Legacy.Gen.u1Long
1213	&& DescCS.Legacy.Gen.u1DefBig)
1214	{
1215	Log(("BranchCallGate uNewCS %04x -> both L and D are set.\n", uNewCS));
1216	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
1217	}
1218	}
1219
1220	if (!DescCS.Legacy.Gate.u1Present)
1221	{
1222	Log(("BranchCallGate target CS is not present. uSel=%04x uNewCS=%04x -> #NP(CS)\n", uSel, uNewCS));
1223	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCS);
1224	}
1225
1226	if (enmBranch == IEMBRANCH_JUMP)
1227	{
1228	/** @todo This is very similar to regular far jumps; merge! */
1229	/* Jumps are fairly simple... */
1230
1231	/* Chop the high bits off if 16-bit gate (Intel says so). */
1232	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
1233	uNewRip = (uint16_t)uNewRip;
1234
1235	/* Limit check for non-long segments. */
1236	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
1237	if (DescCS.Legacy.Gen.u1Long)
1238	u64Base = 0;
1239	else
1240	{
1241	if (uNewRip > cbLimit)
1242	{
1243	Log(("BranchCallGate jump %04x:%08RX64 -> out of bounds (%#x) -> #GP(0)\n", uNewCS, uNewRip, cbLimit));
1244	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
1245	}
1246	u64Base = X86DESC_BASE(&DescCS.Legacy);
1247	}
1248
1249	/* Canonical address check. */
1250	if (!IEM_IS_CANONICAL(uNewRip))
1251	{
1252	Log(("BranchCallGate jump %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
1253	return iemRaiseNotCanonical(pVCpu);
1254	}
1255
1256	/*
1257	* Ok, everything checked out fine. Now set the accessed bit before
1258	* committing the result into CS, CSHID and RIP.
1259	*/
1260	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1261	{
1262	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
1263	if (rcStrict != VINF_SUCCESS)
1264	return rcStrict;
1265	/** @todo check what VT-x and AMD-V does. */
1266	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1267	}
1268
1269	/* commit */
1270	pVCpu->cpum.GstCtx.rip = uNewRip;
1271	pVCpu->cpum.GstCtx.cs.Sel = uNewCS & X86_SEL_MASK_OFF_RPL;
1272	pVCpu->cpum.GstCtx.cs.Sel \|= pVCpu->iem.s.uCpl; /** @todo is this right for conforming segs? or in general? */
1273	pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
1274	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1275	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
1276	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
1277	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
1278	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
1279	}
1280	else
1281	{
1282	Assert(enmBranch == IEMBRANCH_CALL);
1283	/* Calls are much more complicated. */
1284
1285	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) && (DescCS.Legacy.Gen.u2Dpl < pVCpu->iem.s.uCpl))
1286	{
1287	uint16_t offNewStack; /* Offset of new stack in TSS. */
1288	uint16_t cbNewStack; /* Number of bytes the stack information takes up in TSS. */
1289	uint8_t uNewCSDpl;
1290	uint8_t cbWords;
1291	RTSEL uNewSS;
1292	RTSEL uOldSS;
1293	uint64_t uOldRsp;
1294	IEMSELDESC DescSS;
1295	RTPTRUNION uPtrTSS;
1296	RTGCPTR GCPtrTSS;
1297	RTPTRUNION uPtrParmWds;
1298	RTGCPTR GCPtrParmWds;
1299
1300	/* More privilege. This is the fun part. */
1301	Assert(!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)); /* Filtered out above. */
1302
1303	/*
1304	* Determine new SS:rSP from the TSS.
1305	*/
1306	Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType);
1307
1308	/* Figure out where the new stack pointer is stored in the TSS. */
1309	uNewCSDpl = DescCS.Legacy.Gen.u2Dpl;
1310	if (!IEM_IS_LONG_MODE(pVCpu))
1311	{
1312	if (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY)
1313	{
1314	offNewStack = RT_UOFFSETOF(X86TSS32, esp0) + uNewCSDpl * 8;
1315	cbNewStack = RT_SIZEOFMEMB(X86TSS32, esp0) + RT_SIZEOFMEMB(X86TSS32, ss0);
1316	}
1317	else
1318	{
1319	Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY);
1320	offNewStack = RT_UOFFSETOF(X86TSS16, sp0) + uNewCSDpl * 4;
1321	cbNewStack = RT_SIZEOFMEMB(X86TSS16, sp0) + RT_SIZEOFMEMB(X86TSS16, ss0);
1322	}
1323	}
1324	else
1325	{
1326	Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY);
1327	offNewStack = RT_UOFFSETOF(X86TSS64, rsp0) + uNewCSDpl * RT_SIZEOFMEMB(X86TSS64, rsp0);
1328	cbNewStack = RT_SIZEOFMEMB(X86TSS64, rsp0);
1329	}
1330
1331	/* Check against TSS limit. */
1332	if ((uint16_t)(offNewStack + cbNewStack - 1) > pVCpu->cpum.GstCtx.tr.u32Limit)
1333	{
1334	Log(("BranchCallGate inner stack past TSS limit - %u > %u -> #TS(TSS)\n", offNewStack + cbNewStack - 1, pVCpu->cpum.GstCtx.tr.u32Limit));
1335	return iemRaiseTaskSwitchFaultBySelector(pVCpu, pVCpu->cpum.GstCtx.tr.Sel);
1336	}
1337
1338	GCPtrTSS = pVCpu->cpum.GstCtx.tr.u64Base + offNewStack;
1339	rcStrict = iemMemMap(pVCpu, &uPtrTSS.pv, cbNewStack, UINT8_MAX, GCPtrTSS, IEM_ACCESS_SYS_R, 0);
1340	if (rcStrict != VINF_SUCCESS)
1341	{
1342	Log(("BranchCallGate: TSS mapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1343	return rcStrict;
1344	}
1345
1346	if (!IEM_IS_LONG_MODE(pVCpu))
1347	{
1348	if (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY)
1349	{
1350	uNewRsp = uPtrTSS.pu32[0];
1351	uNewSS = uPtrTSS.pu16[2];
1352	}
1353	else
1354	{
1355	Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY);
1356	uNewRsp = uPtrTSS.pu16[0];
1357	uNewSS = uPtrTSS.pu16[1];
1358	}
1359	}
1360	else
1361	{
1362	Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY);
1363	/* SS will be a NULL selector, but that's valid. */
1364	uNewRsp = uPtrTSS.pu64[0];
1365	uNewSS = uNewCSDpl;
1366	}
1367
1368	/* Done with the TSS now. */
1369	rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrTSS.pv, IEM_ACCESS_SYS_R);
1370	if (rcStrict != VINF_SUCCESS)
1371	{
1372	Log(("BranchCallGate: TSS unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1373	return rcStrict;
1374	}
1375
1376	/* Only used outside of long mode. */
1377	cbWords = pDesc->Legacy.Gate.u5ParmCount;
1378
1379	/* If EFER.LMA is 0, there's extra work to do. */
1380	if (!IEM_IS_LONG_MODE(pVCpu))
1381	{
1382	if ((uNewSS & X86_SEL_MASK_OFF_RPL) == 0)
1383	{
1384	Log(("BranchCallGate new SS NULL -> #TS(NewSS)\n"));
1385	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
1386	}
1387
1388	/* Grab the new SS descriptor. */
1389	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_SS);
1390	if (rcStrict != VINF_SUCCESS)
1391	return rcStrict;
1392
1393	/* Ensure that CS.DPL == SS.RPL == SS.DPL. */
1394	if ( (DescCS.Legacy.Gen.u2Dpl != (uNewSS & X86_SEL_RPL))
1395	\|\| (DescCS.Legacy.Gen.u2Dpl != DescSS.Legacy.Gen.u2Dpl))
1396	{
1397	Log(("BranchCallGate call bad RPL/DPL uNewSS=%04x SS DPL=%d CS DPL=%u -> #TS(NewSS)\n",
1398	uNewSS, DescCS.Legacy.Gen.u2Dpl, DescCS.Legacy.Gen.u2Dpl));
1399	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
1400	}
1401
1402	/* Ensure new SS is a writable data segment. */
1403	if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
1404	{
1405	Log(("BranchCallGate call new SS -> not a writable data selector (u4Type=%#x)\n", DescSS.Legacy.Gen.u4Type));
1406	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
1407	}
1408
1409	if (!DescSS.Legacy.Gen.u1Present)
1410	{
1411	Log(("BranchCallGate New stack not present uSel=%04x -> #SS(NewSS)\n", uNewSS));
1412	return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSS);
1413	}
1414	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
1415	cbNewStack = (uint16_t)sizeof(uint32_t) * (4 + cbWords);
1416	else
1417	cbNewStack = (uint16_t)sizeof(uint16_t) * (4 + cbWords);
1418	}
1419	else
1420	{
1421	/* Just grab the new (NULL) SS descriptor. */
1422	/** @todo testcase: Check whether the zero GDT entry is actually loaded here
1423	* like we do... */
1424	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_SS);
1425	if (rcStrict != VINF_SUCCESS)
1426	return rcStrict;
1427
1428	cbNewStack = sizeof(uint64_t) * 4;
1429	}
1430
1431	/** @todo According to Intel, new stack is checked for enough space first,
1432	* then switched. According to AMD, the stack is switched first and
1433	* then pushes might fault!
1434	* NB: OS/2 Warp 3/4 actively relies on the fact that possible
1435	* incoming stack \#PF happens before actual stack switch. AMD is
1436	* either lying or implicitly assumes that new state is committed
1437	* only if and when an instruction doesn't fault.
1438	*/
1439
1440	/** @todo According to AMD, CS is loaded first, then SS.
1441	* According to Intel, it's the other way around!?
1442	*/
1443
1444	/** @todo Intel and AMD disagree on when exactly the CPL changes! */
1445
1446	/* Set the accessed bit before committing new SS. */
1447	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1448	{
1449	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
1450	if (rcStrict != VINF_SUCCESS)
1451	return rcStrict;
1452	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1453	}
1454
1455	/* Remember the old SS:rSP and their linear address. */
1456	uOldSS = pVCpu->cpum.GstCtx.ss.Sel;
1457	uOldRsp = pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig ? pVCpu->cpum.GstCtx.rsp : pVCpu->cpum.GstCtx.sp;
1458
1459	GCPtrParmWds = pVCpu->cpum.GstCtx.ss.u64Base + uOldRsp;
1460
1461	/* HACK ALERT! Probe if the write to the new stack will succeed. May #SS(NewSS)
1462	or #PF, the former is not implemented in this workaround. */
1463	/** @todo Proper fix callgate target stack exceptions. */
1464	/** @todo testcase: Cover callgates with partially or fully inaccessible
1465	* target stacks. */
1466	void *pvNewFrame;
1467	RTGCPTR GCPtrNewStack = X86DESC_BASE(&DescSS.Legacy) + uNewRsp - cbNewStack;
1468	rcStrict = iemMemMap(pVCpu, &pvNewFrame, cbNewStack, UINT8_MAX, GCPtrNewStack, IEM_ACCESS_SYS_RW, 0);
1469	if (rcStrict != VINF_SUCCESS)
1470	{
1471	Log(("BranchCallGate: Incoming stack (%04x:%08RX64) not accessible, rc=%Rrc\n", uNewSS, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
1472	return rcStrict;
1473	}
1474	rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewFrame, IEM_ACCESS_SYS_RW);
1475	if (rcStrict != VINF_SUCCESS)
1476	{
1477	Log(("BranchCallGate: New stack probe unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1478	return rcStrict;
1479	}
1480
1481	/* Commit new SS:rSP. */
1482	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
1483	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
1484	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
1485	pVCpu->cpum.GstCtx.ss.u32Limit = X86DESC_LIMIT_G(&DescSS.Legacy);
1486	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
1487	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
1488	pVCpu->cpum.GstCtx.rsp = uNewRsp;
1489	pVCpu->iem.s.uCpl = uNewCSDpl; /** @todo is the parameter words accessed using the new CPL or the old CPL? */
1490	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1491	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
1492
1493	/* At this point the stack access must not fail because new state was already committed. */
1494	/** @todo this can still fail due to SS.LIMIT not check. */
1495	rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbNewStack,
1496	IEM_IS_LONG_MODE(pVCpu) ? 7
1497	: pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 3 : 1,
1498	&uPtrRet.pv, &uNewRsp);
1499	AssertMsgReturn(rcStrict == VINF_SUCCESS, ("BranchCallGate: New stack mapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)),
1500	VERR_INTERNAL_ERROR_5);
1501
1502	if (!IEM_IS_LONG_MODE(pVCpu))
1503	{
1504	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
1505	{
1506	if (cbWords)
1507	{
1508	/* Map the relevant chunk of the old stack. */
1509	rcStrict = iemMemMap(pVCpu, &uPtrParmWds.pv, cbWords * 4, UINT8_MAX, GCPtrParmWds,
1510	IEM_ACCESS_DATA_R, 0 /** @todo Can uNewCSDpl == 3? Then we need alignment mask here! */);
1511	if (rcStrict != VINF_SUCCESS)
1512	{
1513	Log(("BranchCallGate: Old stack mapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1514	return rcStrict;
1515	}
1516
1517	/* Copy the parameter (d)words. */
1518	for (int i = 0; i < cbWords; ++i)
1519	uPtrRet.pu32[2 + i] = uPtrParmWds.pu32[i];
1520
1521	/* Unmap the old stack. */
1522	rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrParmWds.pv, IEM_ACCESS_DATA_R);
1523	if (rcStrict != VINF_SUCCESS)
1524	{
1525	Log(("BranchCallGate: Old stack unmapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1526	return rcStrict;
1527	}
1528	}
1529
1530	/* Push the old CS:rIP. */
1531	uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
1532	uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when pushing CS? */
1533
1534	/* Push the old SS:rSP. */
1535	uPtrRet.pu32[2 + cbWords + 0] = uOldRsp;
1536	uPtrRet.pu32[2 + cbWords + 1] = uOldSS;
1537	}
1538	else
1539	{
1540	Assert(pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE);
1541
1542	if (cbWords)
1543	{
1544	/* Map the relevant chunk of the old stack. */
1545	rcStrict = iemMemMap(pVCpu, &uPtrParmWds.pv, cbWords * 2, UINT8_MAX, GCPtrParmWds,
1546	IEM_ACCESS_DATA_R, 0 /** @todo Can uNewCSDpl == 3? Then we need alignment mask here! */);
1547	if (rcStrict != VINF_SUCCESS)
1548	{
1549	Log(("BranchCallGate: Old stack mapping (16-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1550	return rcStrict;
1551	}
1552
1553	/* Copy the parameter words. */
1554	for (int i = 0; i < cbWords; ++i)
1555	uPtrRet.pu16[2 + i] = uPtrParmWds.pu16[i];
1556
1557	/* Unmap the old stack. */
1558	rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrParmWds.pv, IEM_ACCESS_DATA_R);
1559	if (rcStrict != VINF_SUCCESS)
1560	{
1561	Log(("BranchCallGate: Old stack unmapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1562	return rcStrict;
1563	}
1564	}
1565
1566	/* Push the old CS:rIP. */
1567	uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
1568	uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
1569
1570	/* Push the old SS:rSP. */
1571	uPtrRet.pu16[2 + cbWords + 0] = uOldRsp;
1572	uPtrRet.pu16[2 + cbWords + 1] = uOldSS;
1573	}
1574	}
1575	else
1576	{
1577	Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
1578
1579	/* For 64-bit gates, no parameters are copied. Just push old SS:rSP and CS:rIP. */
1580	uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
1581	uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when pushing CS? */
1582	uPtrRet.pu64[2] = uOldRsp;
1583	uPtrRet.pu64[3] = uOldSS; /** @todo Testcase: What is written to the high words when pushing SS? */
1584	}
1585
1586	rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
1587	if (rcStrict != VINF_SUCCESS)
1588	{
1589	Log(("BranchCallGate: New stack unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1590	return rcStrict;
1591	}
1592
1593	/* Chop the high bits off if 16-bit gate (Intel says so). */
1594	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
1595	uNewRip = (uint16_t)uNewRip;
1596
1597	/* Limit / canonical check. */
1598	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
1599	if (!IEM_IS_LONG_MODE(pVCpu))
1600	{
1601	if (uNewRip > cbLimit)
1602	{
1603	Log(("BranchCallGate %04x:%08RX64 -> out of bounds (%#x)\n", uNewCS, uNewRip, cbLimit));
1604	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
1605	}
1606	u64Base = X86DESC_BASE(&DescCS.Legacy);
1607	}
1608	else
1609	{
1610	Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
1611	if (!IEM_IS_CANONICAL(uNewRip))
1612	{
1613	Log(("BranchCallGate call %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
1614	return iemRaiseNotCanonical(pVCpu);
1615	}
1616	u64Base = 0;
1617	}
1618
1619	/*
1620	* Now set the accessed bit before
1621	* writing the return address to the stack and committing the result into
1622	* CS, CSHID and RIP.
1623	*/
1624	/** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
1625	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1626	{
1627	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
1628	if (rcStrict != VINF_SUCCESS)
1629	return rcStrict;
1630	/** @todo check what VT-x and AMD-V does. */
1631	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1632	}
1633
1634	/* Commit new CS:rIP. */
1635	pVCpu->cpum.GstCtx.rip = uNewRip;
1636	pVCpu->cpum.GstCtx.cs.Sel = uNewCS & X86_SEL_MASK_OFF_RPL;
1637	pVCpu->cpum.GstCtx.cs.Sel \|= pVCpu->iem.s.uCpl;
1638	pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
1639	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1640	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
1641	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
1642	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
1643	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
1644	}
1645	else
1646	{
1647	/* Same privilege. */
1648	/** @todo This is very similar to regular far calls; merge! */
1649
1650	/* Check stack first - may #SS(0). */
1651	/** @todo check how gate size affects pushing of CS! Does callf 16:32 in
1652	* 16-bit code cause a two or four byte CS to be pushed? */
1653	rcStrict = iemMemStackPushBeginSpecial(pVCpu,
1654	IEM_IS_LONG_MODE(pVCpu) ? 8+8
1655	: pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 4+4 : 2+2,
1656	IEM_IS_LONG_MODE(pVCpu) ? 7
1657	: pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 3 : 2,
1658	&uPtrRet.pv, &uNewRsp);
1659	if (rcStrict != VINF_SUCCESS)
1660	return rcStrict;
1661
1662	/* Chop the high bits off if 16-bit gate (Intel says so). */
1663	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
1664	uNewRip = (uint16_t)uNewRip;
1665
1666	/* Limit / canonical check. */
1667	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
1668	if (!IEM_IS_LONG_MODE(pVCpu))
1669	{
1670	if (uNewRip > cbLimit)
1671	{
1672	Log(("BranchCallGate %04x:%08RX64 -> out of bounds (%#x)\n", uNewCS, uNewRip, cbLimit));
1673	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
1674	}
1675	u64Base = X86DESC_BASE(&DescCS.Legacy);
1676	}
1677	else
1678	{
1679	if (!IEM_IS_CANONICAL(uNewRip))
1680	{
1681	Log(("BranchCallGate call %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
1682	return iemRaiseNotCanonical(pVCpu);
1683	}
1684	u64Base = 0;
1685	}
1686
1687	/*
1688	* Now set the accessed bit before
1689	* writing the return address to the stack and committing the result into
1690	* CS, CSHID and RIP.
1691	*/
1692	/** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
1693	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1694	{
1695	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
1696	if (rcStrict != VINF_SUCCESS)
1697	return rcStrict;
1698	/** @todo check what VT-x and AMD-V does. */
1699	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1700	}
1701
1702	/* stack */
1703	if (!IEM_IS_LONG_MODE(pVCpu))
1704	{
1705	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
1706	{
1707	uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
1708	uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when pushing CS? */
1709	}
1710	else
1711	{
1712	Assert(pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE);
1713	uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
1714	uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
1715	}
1716	}
1717	else
1718	{
1719	Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
1720	uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
1721	uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when pushing CS? */
1722	}
1723
1724	rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
1725	if (rcStrict != VINF_SUCCESS)
1726	return rcStrict;
1727
1728	/* commit */
1729	pVCpu->cpum.GstCtx.rip = uNewRip;
1730	pVCpu->cpum.GstCtx.cs.Sel = uNewCS & X86_SEL_MASK_OFF_RPL;
1731	pVCpu->cpum.GstCtx.cs.Sel \|= pVCpu->iem.s.uCpl;
1732	pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
1733	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1734	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
1735	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
1736	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
1737	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
1738	}
1739	}
1740	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
1741
1742	/* Flush the prefetch buffer. */
1743	# ifdef IEM_WITH_CODE_TLB
1744	pVCpu->iem.s.pbInstrBuf = NULL;
1745	# else
1746	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
1747	# endif
1748	return VINF_SUCCESS;
1749	#endif
1750	}
1751
1752
1753	/**
1754	* Implements far jumps and calls thru system selectors.
1755	*
1756	* @param uSel The selector.
1757	* @param enmBranch The kind of branching we're performing.
1758	* @param enmEffOpSize The effective operand size.
1759	* @param pDesc The descriptor corresponding to @a uSel.
1760	*/
1761	IEM_CIMPL_DEF_4(iemCImpl_BranchSysSel, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc)
1762	{
1763	Assert(enmBranch == IEMBRANCH_JUMP \|\| enmBranch == IEMBRANCH_CALL);
1764	Assert((uSel & X86_SEL_MASK_OFF_RPL));
1765	IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
1766
1767	if (IEM_IS_LONG_MODE(pVCpu))
1768	switch (pDesc->Legacy.Gen.u4Type)
1769	{
1770	case AMD64_SEL_TYPE_SYS_CALL_GATE:
1771	return IEM_CIMPL_CALL_4(iemCImpl_BranchCallGate, uSel, enmBranch, enmEffOpSize, pDesc);
1772
1773	default:
1774	case AMD64_SEL_TYPE_SYS_LDT:
1775	case AMD64_SEL_TYPE_SYS_TSS_BUSY:
1776	case AMD64_SEL_TYPE_SYS_TSS_AVAIL:
1777	case AMD64_SEL_TYPE_SYS_TRAP_GATE:
1778	case AMD64_SEL_TYPE_SYS_INT_GATE:
1779	Log(("branch %04x -> wrong sys selector (64-bit): %d\n", uSel, pDesc->Legacy.Gen.u4Type));
1780	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1781	}
1782
1783	switch (pDesc->Legacy.Gen.u4Type)
1784	{
1785	case X86_SEL_TYPE_SYS_286_CALL_GATE:
1786	case X86_SEL_TYPE_SYS_386_CALL_GATE:
1787	return IEM_CIMPL_CALL_4(iemCImpl_BranchCallGate, uSel, enmBranch, enmEffOpSize, pDesc);
1788
1789	case X86_SEL_TYPE_SYS_TASK_GATE:
1790	return IEM_CIMPL_CALL_4(iemCImpl_BranchTaskGate, uSel, enmBranch, enmEffOpSize, pDesc);
1791
1792	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
1793	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
1794	return IEM_CIMPL_CALL_4(iemCImpl_BranchTaskSegment, uSel, enmBranch, enmEffOpSize, pDesc);
1795
1796	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1797	Log(("branch %04x -> busy 286 TSS\n", uSel));
1798	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1799
1800	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1801	Log(("branch %04x -> busy 386 TSS\n", uSel));
1802	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1803
1804	default:
1805	case X86_SEL_TYPE_SYS_LDT:
1806	case X86_SEL_TYPE_SYS_286_INT_GATE:
1807	case X86_SEL_TYPE_SYS_286_TRAP_GATE:
1808	case X86_SEL_TYPE_SYS_386_INT_GATE:
1809	case X86_SEL_TYPE_SYS_386_TRAP_GATE:
1810	Log(("branch %04x -> wrong sys selector: %d\n", uSel, pDesc->Legacy.Gen.u4Type));
1811	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1812	}
1813	}
1814
1815
1816	/**
1817	* Implements far jumps.
1818	*
1819	* @param uSel The selector.
1820	* @param offSeg The segment offset.
1821	* @param enmEffOpSize The effective operand size.
1822	*/
1823	IEM_CIMPL_DEF_3(iemCImpl_FarJmp, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize)
1824	{
1825	NOREF(cbInstr);
1826	Assert(offSeg <= UINT32_MAX);
1827
1828	/*
1829	* Real mode and V8086 mode are easy. The only snag seems to be that
1830	* CS.limit doesn't change and the limit check is done against the current
1831	* limit.
1832	*/
1833	/** @todo Robert Collins claims (The Segment Descriptor Cache, DDJ August
1834	* 1998) that up to and including the Intel 486, far control
1835	* transfers in real mode set default CS attributes (0x93) and also
1836	* set a 64K segment limit. Starting with the Pentium, the
1837	* attributes and limit are left alone but the access rights are
1838	* ignored. We only implement the Pentium+ behavior.
1839	* */
1840	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
1841	{
1842	Assert(enmEffOpSize == IEMMODE_16BIT \|\| enmEffOpSize == IEMMODE_32BIT);
1843	if (offSeg > pVCpu->cpum.GstCtx.cs.u32Limit)
1844	{
1845	Log(("iemCImpl_FarJmp: 16-bit limit\n"));
1846	return iemRaiseGeneralProtectionFault0(pVCpu);
1847	}
1848
1849	if (enmEffOpSize == IEMMODE_16BIT) /** @todo WRONG, must pass this. */
1850	pVCpu->cpum.GstCtx.rip = offSeg;
1851	else
1852	pVCpu->cpum.GstCtx.rip = offSeg & UINT16_MAX;
1853	pVCpu->cpum.GstCtx.cs.Sel = uSel;
1854	pVCpu->cpum.GstCtx.cs.ValidSel = uSel;
1855	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1856	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uSel << 4;
1857	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
1858	return VINF_SUCCESS;
1859	}
1860
1861	/*
1862	* Protected mode. Need to parse the specified descriptor...
1863	*/
1864	if (!(uSel & X86_SEL_MASK_OFF_RPL))
1865	{
1866	Log(("jmpf %04x:%08RX64 -> invalid selector, #GP(0)\n", uSel, offSeg));
1867	return iemRaiseGeneralProtectionFault0(pVCpu);
1868	}
1869
1870	/* Fetch the descriptor. */
1871	IEMSELDESC Desc;
1872	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP);
1873	if (rcStrict != VINF_SUCCESS)
1874	return rcStrict;
1875
1876	/* Is it there? */
1877	if (!Desc.Legacy.Gen.u1Present) /** @todo this is probably checked too early. Testcase! */
1878	{
1879	Log(("jmpf %04x:%08RX64 -> segment not present\n", uSel, offSeg));
1880	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
1881	}
1882
1883	/*
1884	* Deal with it according to its type. We do the standard code selectors
1885	* here and dispatch the system selectors to worker functions.
1886	*/
1887	if (!Desc.Legacy.Gen.u1DescType)
1888	return IEM_CIMPL_CALL_4(iemCImpl_BranchSysSel, uSel, IEMBRANCH_JUMP, enmEffOpSize, &Desc);
1889
1890	/* Only code segments. */
1891	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
1892	{
1893	Log(("jmpf %04x:%08RX64 -> not a code selector (u4Type=%#x).\n", uSel, offSeg, Desc.Legacy.Gen.u4Type));
1894	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1895	}
1896
1897	/* L vs D. */
1898	if ( Desc.Legacy.Gen.u1Long
1899	&& Desc.Legacy.Gen.u1DefBig
1900	&& IEM_IS_LONG_MODE(pVCpu))
1901	{
1902	Log(("jmpf %04x:%08RX64 -> both L and D are set.\n", uSel, offSeg));
1903	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1904	}
1905
1906	/* DPL/RPL/CPL check, where conforming segments makes a difference. */
1907	if (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
1908	{
1909	if (pVCpu->iem.s.uCpl < Desc.Legacy.Gen.u2Dpl)
1910	{
1911	Log(("jmpf %04x:%08RX64 -> DPL violation (conforming); DPL=%d CPL=%u\n",
1912	uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
1913	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1914	}
1915	}
1916	else
1917	{
1918	if (pVCpu->iem.s.uCpl != Desc.Legacy.Gen.u2Dpl)
1919	{
1920	Log(("jmpf %04x:%08RX64 -> CPL != DPL; DPL=%d CPL=%u\n", uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
1921	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1922	}
1923	if ((uSel & X86_SEL_RPL) > pVCpu->iem.s.uCpl)
1924	{
1925	Log(("jmpf %04x:%08RX64 -> RPL > DPL; RPL=%d CPL=%u\n", uSel, offSeg, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl));
1926	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1927	}
1928	}
1929
1930	/* Chop the high bits if 16-bit (Intel says so). */
1931	if (enmEffOpSize == IEMMODE_16BIT)
1932	offSeg &= UINT16_MAX;
1933
1934	/* Limit check. (Should alternatively check for non-canonical addresses
1935	here, but that is ruled out by offSeg being 32-bit, right?) */
1936	uint64_t u64Base;
1937	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
1938	if (Desc.Legacy.Gen.u1Long)
1939	u64Base = 0;
1940	else
1941	{
1942	if (offSeg > cbLimit)
1943	{
1944	Log(("jmpf %04x:%08RX64 -> out of bounds (%#x)\n", uSel, offSeg, cbLimit));
1945	/** @todo Intel says this is \#GP(0)! */
1946	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1947	}
1948	u64Base = X86DESC_BASE(&Desc.Legacy);
1949	}
1950
1951	/*
1952	* Ok, everything checked out fine. Now set the accessed bit before
1953	* committing the result into CS, CSHID and RIP.
1954	*/
1955	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1956	{
1957	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
1958	if (rcStrict != VINF_SUCCESS)
1959	return rcStrict;
1960	/** @todo check what VT-x and AMD-V does. */
1961	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1962	}
1963
1964	/* commit */
1965	pVCpu->cpum.GstCtx.rip = offSeg;
1966	pVCpu->cpum.GstCtx.cs.Sel = uSel & X86_SEL_MASK_OFF_RPL;
1967	pVCpu->cpum.GstCtx.cs.Sel \|= pVCpu->iem.s.uCpl; /** @todo is this right for conforming segs? or in general? */
1968	pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
1969	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1970	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
1971	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
1972	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
1973	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
1974	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
1975	/** @todo check if the hidden bits are loaded correctly for 64-bit
1976	* mode. */
1977
1978	/* Flush the prefetch buffer. */
1979	#ifdef IEM_WITH_CODE_TLB
1980	pVCpu->iem.s.pbInstrBuf = NULL;
1981	#else
1982	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
1983	#endif
1984
1985	return VINF_SUCCESS;
1986	}
1987
1988
1989	/**
1990	* Implements far calls.
1991	*
1992	* This very similar to iemCImpl_FarJmp.
1993	*
1994	* @param uSel The selector.
1995	* @param offSeg The segment offset.
1996	* @param enmEffOpSize The operand size (in case we need it).
1997	*/
1998	IEM_CIMPL_DEF_3(iemCImpl_callf, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize)
1999	{
2000	VBOXSTRICTRC rcStrict;
2001	uint64_t uNewRsp;
2002	RTPTRUNION uPtrRet;
2003
2004	/*
2005	* Real mode and V8086 mode are easy. The only snag seems to be that
2006	* CS.limit doesn't change and the limit check is done against the current
2007	* limit.
2008	*/
2009	/** @todo See comment for similar code in iemCImpl_FarJmp */
2010	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2011	{
2012	Assert(enmEffOpSize == IEMMODE_16BIT \|\| enmEffOpSize == IEMMODE_32BIT);
2013
2014	/* Check stack first - may #SS(0). */
2015	rcStrict = iemMemStackPushBeginSpecial(pVCpu, enmEffOpSize == IEMMODE_32BIT ? 4+4 : 2+2,
2016	enmEffOpSize == IEMMODE_32BIT ? 3 : 1,
2017	&uPtrRet.pv, &uNewRsp);
2018	if (rcStrict != VINF_SUCCESS)
2019	return rcStrict;
2020
2021	/* Check the target address range. */
2022	if (offSeg > UINT32_MAX)
2023	return iemRaiseGeneralProtectionFault0(pVCpu);
2024
2025	/* Everything is fine, push the return address. */
2026	if (enmEffOpSize == IEMMODE_16BIT)
2027	{
2028	uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
2029	uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
2030	}
2031	else
2032	{
2033	uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
2034	uPtrRet.pu16[2] = pVCpu->cpum.GstCtx.cs.Sel;
2035	}
2036	rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
2037	if (rcStrict != VINF_SUCCESS)
2038	return rcStrict;
2039
2040	/* Branch. */
2041	pVCpu->cpum.GstCtx.rip = offSeg;
2042	pVCpu->cpum.GstCtx.cs.Sel = uSel;
2043	pVCpu->cpum.GstCtx.cs.ValidSel = uSel;
2044	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2045	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uSel << 4;
2046	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
2047	return VINF_SUCCESS;
2048	}
2049
2050	/*
2051	* Protected mode. Need to parse the specified descriptor...
2052	*/
2053	if (!(uSel & X86_SEL_MASK_OFF_RPL))
2054	{
2055	Log(("callf %04x:%08RX64 -> invalid selector, #GP(0)\n", uSel, offSeg));
2056	return iemRaiseGeneralProtectionFault0(pVCpu);
2057	}
2058
2059	/* Fetch the descriptor. */
2060	IEMSELDESC Desc;
2061	rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP);
2062	if (rcStrict != VINF_SUCCESS)
2063	return rcStrict;
2064
2065	/*
2066	* Deal with it according to its type. We do the standard code selectors
2067	* here and dispatch the system selectors to worker functions.
2068	*/
2069	if (!Desc.Legacy.Gen.u1DescType)
2070	return IEM_CIMPL_CALL_4(iemCImpl_BranchSysSel, uSel, IEMBRANCH_CALL, enmEffOpSize, &Desc);
2071
2072	/* Only code segments. */
2073	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
2074	{
2075	Log(("callf %04x:%08RX64 -> not a code selector (u4Type=%#x).\n", uSel, offSeg, Desc.Legacy.Gen.u4Type));
2076	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2077	}
2078
2079	/* L vs D. */
2080	if ( Desc.Legacy.Gen.u1Long
2081	&& Desc.Legacy.Gen.u1DefBig
2082	&& IEM_IS_LONG_MODE(pVCpu))
2083	{
2084	Log(("callf %04x:%08RX64 -> both L and D are set.\n", uSel, offSeg));
2085	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2086	}
2087
2088	/* DPL/RPL/CPL check, where conforming segments makes a difference. */
2089	if (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2090	{
2091	if (pVCpu->iem.s.uCpl < Desc.Legacy.Gen.u2Dpl)
2092	{
2093	Log(("callf %04x:%08RX64 -> DPL violation (conforming); DPL=%d CPL=%u\n",
2094	uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
2095	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2096	}
2097	}
2098	else
2099	{
2100	if (pVCpu->iem.s.uCpl != Desc.Legacy.Gen.u2Dpl)
2101	{
2102	Log(("callf %04x:%08RX64 -> CPL != DPL; DPL=%d CPL=%u\n", uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
2103	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2104	}
2105	if ((uSel & X86_SEL_RPL) > pVCpu->iem.s.uCpl)
2106	{
2107	Log(("callf %04x:%08RX64 -> RPL > DPL; RPL=%d CPL=%u\n", uSel, offSeg, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl));
2108	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2109	}
2110	}
2111
2112	/* Is it there? */
2113	if (!Desc.Legacy.Gen.u1Present)
2114	{
2115	Log(("callf %04x:%08RX64 -> segment not present\n", uSel, offSeg));
2116	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
2117	}
2118
2119	/* Check stack first - may #SS(0). */
2120	/** @todo check how operand prefix affects pushing of CS! Does callf 16:32 in
2121	* 16-bit code cause a two or four byte CS to be pushed? */
2122	rcStrict = iemMemStackPushBeginSpecial(pVCpu,
2123	enmEffOpSize == IEMMODE_64BIT ? 8+8 : enmEffOpSize == IEMMODE_32BIT ? 4+4 : 2+2,
2124	enmEffOpSize == IEMMODE_64BIT ? 7 : enmEffOpSize == IEMMODE_32BIT ? 3 : 1,
2125	&uPtrRet.pv, &uNewRsp);
2126	if (rcStrict != VINF_SUCCESS)
2127	return rcStrict;
2128
2129	/* Chop the high bits if 16-bit (Intel says so). */
2130	if (enmEffOpSize == IEMMODE_16BIT)
2131	offSeg &= UINT16_MAX;
2132
2133	/* Limit / canonical check. */
2134	uint64_t u64Base;
2135	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
2136	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2137	{
2138	if (!IEM_IS_CANONICAL(offSeg))
2139	{
2140	Log(("callf %04x:%016RX64 - not canonical -> #GP\n", uSel, offSeg));
2141	return iemRaiseNotCanonical(pVCpu);
2142	}
2143	u64Base = 0;
2144	}
2145	else
2146	{
2147	if (offSeg > cbLimit)
2148	{
2149	Log(("callf %04x:%08RX64 -> out of bounds (%#x)\n", uSel, offSeg, cbLimit));
2150	/** @todo Intel says this is \#GP(0)! */
2151	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2152	}
2153	u64Base = X86DESC_BASE(&Desc.Legacy);
2154	}
2155
2156	/*
2157	* Now set the accessed bit before
2158	* writing the return address to the stack and committing the result into
2159	* CS, CSHID and RIP.
2160	*/
2161	/** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
2162	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2163	{
2164	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
2165	if (rcStrict != VINF_SUCCESS)
2166	return rcStrict;
2167	/** @todo check what VT-x and AMD-V does. */
2168	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2169	}
2170
2171	/* stack */
2172	if (enmEffOpSize == IEMMODE_16BIT)
2173	{
2174	uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
2175	uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
2176	}
2177	else if (enmEffOpSize == IEMMODE_32BIT)
2178	{
2179	uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
2180	uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when callf is pushing CS? */
2181	}
2182	else
2183	{
2184	uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
2185	uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when callf is pushing CS? */
2186	}
2187	rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
2188	if (rcStrict != VINF_SUCCESS)
2189	return rcStrict;
2190
2191	/* commit */
2192	pVCpu->cpum.GstCtx.rip = offSeg;
2193	pVCpu->cpum.GstCtx.cs.Sel = uSel & X86_SEL_MASK_OFF_RPL;
2194	pVCpu->cpum.GstCtx.cs.Sel \|= pVCpu->iem.s.uCpl;
2195	pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
2196	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2197	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
2198	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
2199	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
2200	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
2201	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
2202	/** @todo check if the hidden bits are loaded correctly for 64-bit
2203	* mode. */
2204
2205	/* Flush the prefetch buffer. */
2206	#ifdef IEM_WITH_CODE_TLB
2207	pVCpu->iem.s.pbInstrBuf = NULL;
2208	#else
2209	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
2210	#endif
2211	return VINF_SUCCESS;
2212	}
2213
2214
2215	/**
2216	* Implements retf.
2217	*
2218	* @param enmEffOpSize The effective operand size.
2219	* @param cbPop The amount of arguments to pop from the stack
2220	* (bytes).
2221	*/
2222	IEM_CIMPL_DEF_2(iemCImpl_retf, IEMMODE, enmEffOpSize, uint16_t, cbPop)
2223	{
2224	VBOXSTRICTRC rcStrict;
2225	RTCPTRUNION uPtrFrame;
2226	uint64_t uNewRsp;
2227	uint64_t uNewRip;
2228	uint16_t uNewCs;
2229	NOREF(cbInstr);
2230
2231	/*
2232	* Read the stack values first.
2233	*/
2234	uint32_t cbRetPtr = enmEffOpSize == IEMMODE_16BIT ? 2+2
2235	: enmEffOpSize == IEMMODE_32BIT ? 4+4 : 8+8;
2236	rcStrict = iemMemStackPopBeginSpecial(pVCpu, cbRetPtr,
2237	enmEffOpSize == IEMMODE_16BIT ? 1 : enmEffOpSize == IEMMODE_32BIT ? 3 : 7,
2238	&uPtrFrame.pv, &uNewRsp);
2239	if (rcStrict != VINF_SUCCESS)
2240	return rcStrict;
2241	if (enmEffOpSize == IEMMODE_16BIT)
2242	{
2243	uNewRip = uPtrFrame.pu16[0];
2244	uNewCs = uPtrFrame.pu16[1];
2245	}
2246	else if (enmEffOpSize == IEMMODE_32BIT)
2247	{
2248	uNewRip = uPtrFrame.pu32[0];
2249	uNewCs = uPtrFrame.pu16[2];
2250	}
2251	else
2252	{
2253	uNewRip = uPtrFrame.pu64[0];
2254	uNewCs = uPtrFrame.pu16[4];
2255	}
2256	rcStrict = iemMemStackPopDoneSpecial(pVCpu, uPtrFrame.pv);
2257	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2258	{ /* extremely likely */ }
2259	else
2260	return rcStrict;
2261
2262	/*
2263	* Real mode and V8086 mode are easy.
2264	*/
2265	/** @todo See comment for similar code in iemCImpl_FarJmp */
2266	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2267	{
2268	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2269	/** @todo check how this is supposed to work if sp=0xfffe. */
2270
2271	/* Check the limit of the new EIP. */
2272	/** @todo Intel pseudo code only does the limit check for 16-bit
2273	* operands, AMD does not make any distinction. What is right? */
2274	if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
2275	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
2276
2277	/* commit the operation. */
2278	pVCpu->cpum.GstCtx.rsp = uNewRsp;
2279	pVCpu->cpum.GstCtx.rip = uNewRip;
2280	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
2281	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
2282	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2283	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uNewCs << 4;
2284	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
2285	if (cbPop)
2286	iemRegAddToRsp(pVCpu, cbPop);
2287	return VINF_SUCCESS;
2288	}
2289
2290	/*
2291	* Protected mode is complicated, of course.
2292	*/
2293	if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
2294	{
2295	Log(("retf %04x:%08RX64 -> invalid selector, #GP(0)\n", uNewCs, uNewRip));
2296	return iemRaiseGeneralProtectionFault0(pVCpu);
2297	}
2298
2299	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
2300
2301	/* Fetch the descriptor. */
2302	IEMSELDESC DescCs;
2303	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCs, uNewCs, X86_XCPT_GP);
2304	if (rcStrict != VINF_SUCCESS)
2305	return rcStrict;
2306
2307	/* Can only return to a code selector. */
2308	if ( !DescCs.Legacy.Gen.u1DescType
2309	\|\| !(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) )
2310	{
2311	Log(("retf %04x:%08RX64 -> not a code selector (u1DescType=%u u4Type=%#x).\n",
2312	uNewCs, uNewRip, DescCs.Legacy.Gen.u1DescType, DescCs.Legacy.Gen.u4Type));
2313	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2314	}
2315
2316	/* L vs D. */
2317	if ( DescCs.Legacy.Gen.u1Long /** @todo Testcase: far return to a selector with both L and D set. */
2318	&& DescCs.Legacy.Gen.u1DefBig
2319	&& IEM_IS_LONG_MODE(pVCpu))
2320	{
2321	Log(("retf %04x:%08RX64 -> both L & D set.\n", uNewCs, uNewRip));
2322	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2323	}
2324
2325	/* DPL/RPL/CPL checks. */
2326	if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl)
2327	{
2328	Log(("retf %04x:%08RX64 -> RPL < CPL(%d).\n", uNewCs, uNewRip, pVCpu->iem.s.uCpl));
2329	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2330	}
2331
2332	if (DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2333	{
2334	if ((uNewCs & X86_SEL_RPL) < DescCs.Legacy.Gen.u2Dpl)
2335	{
2336	Log(("retf %04x:%08RX64 -> DPL violation (conforming); DPL=%u RPL=%u\n",
2337	uNewCs, uNewRip, DescCs.Legacy.Gen.u2Dpl, (uNewCs & X86_SEL_RPL)));
2338	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2339	}
2340	}
2341	else
2342	{
2343	if ((uNewCs & X86_SEL_RPL) != DescCs.Legacy.Gen.u2Dpl)
2344	{
2345	Log(("retf %04x:%08RX64 -> RPL != DPL; DPL=%u RPL=%u\n",
2346	uNewCs, uNewRip, DescCs.Legacy.Gen.u2Dpl, (uNewCs & X86_SEL_RPL)));
2347	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2348	}
2349	}
2350
2351	/* Is it there? */
2352	if (!DescCs.Legacy.Gen.u1Present)
2353	{
2354	Log(("retf %04x:%08RX64 -> segment not present\n", uNewCs, uNewRip));
2355	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
2356	}
2357
2358	/*
2359	* Return to outer privilege? (We'll typically have entered via a call gate.)
2360	*/
2361	if ((uNewCs & X86_SEL_RPL) != pVCpu->iem.s.uCpl)
2362	{
2363	/* Read the outer stack pointer stored after the parameters. */
2364	rcStrict = iemMemStackPopContinueSpecial(pVCpu, cbPop /off/, cbRetPtr, &uPtrFrame.pv, uNewRsp);
2365	if (rcStrict != VINF_SUCCESS)
2366	return rcStrict;
2367
2368	uint16_t uNewOuterSs;
2369	uint64_t uNewOuterRsp;
2370	if (enmEffOpSize == IEMMODE_16BIT)
2371	{
2372	uNewOuterRsp = uPtrFrame.pu16[0];
2373	uNewOuterSs = uPtrFrame.pu16[1];
2374	}
2375	else if (enmEffOpSize == IEMMODE_32BIT)
2376	{
2377	uNewOuterRsp = uPtrFrame.pu32[0];
2378	uNewOuterSs = uPtrFrame.pu16[2];
2379	}
2380	else
2381	{
2382	uNewOuterRsp = uPtrFrame.pu64[0];
2383	uNewOuterSs = uPtrFrame.pu16[4];
2384	}
2385	rcStrict = iemMemStackPopDoneSpecial(pVCpu, uPtrFrame.pv);
2386	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2387	{ /* extremely likely */ }
2388	else
2389	return rcStrict;
2390
2391	/* Check for NULL stack selector (invalid in ring-3 and non-long mode)
2392	and read the selector. */
2393	IEMSELDESC DescSs;
2394	if (!(uNewOuterSs & X86_SEL_MASK_OFF_RPL))
2395	{
2396	if ( !DescCs.Legacy.Gen.u1Long
2397	\|\| (uNewOuterSs & X86_SEL_RPL) == 3)
2398	{
2399	Log(("retf %04x:%08RX64 %04x:%08RX64 -> invalid stack selector, #GP\n",
2400	uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp));
2401	return iemRaiseGeneralProtectionFault0(pVCpu);
2402	}
2403	/** @todo Testcase: Return far to ring-1 or ring-2 with SS=0. */
2404	iemMemFakeStackSelDesc(&DescSs, (uNewOuterSs & X86_SEL_RPL));
2405	}
2406	else
2407	{
2408	/* Fetch the descriptor for the new stack segment. */
2409	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSs, uNewOuterSs, X86_XCPT_GP);
2410	if (rcStrict != VINF_SUCCESS)
2411	return rcStrict;
2412	}
2413
2414	/* Check that RPL of stack and code selectors match. */
2415	if ((uNewCs & X86_SEL_RPL) != (uNewOuterSs & X86_SEL_RPL))
2416	{
2417	Log(("retf %04x:%08RX64 %04x:%08RX64 - SS.RPL != CS.RPL -> #GP(SS)\n", uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp));
2418	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
2419	}
2420
2421	/* Must be a writable data segment. */
2422	if ( !DescSs.Legacy.Gen.u1DescType
2423	\|\| (DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
2424	\|\| !(DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
2425	{
2426	Log(("retf %04x:%08RX64 %04x:%08RX64 - SS not a writable data segment (u1DescType=%u u4Type=%#x) -> #GP(SS).\n",
2427	uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp, DescSs.Legacy.Gen.u1DescType, DescSs.Legacy.Gen.u4Type));
2428	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
2429	}
2430
2431	/* L vs D. (Not mentioned by intel.) */
2432	if ( DescSs.Legacy.Gen.u1Long /** @todo Testcase: far return to a stack selector with both L and D set. */
2433	&& DescSs.Legacy.Gen.u1DefBig
2434	&& IEM_IS_LONG_MODE(pVCpu))
2435	{
2436	Log(("retf %04x:%08RX64 %04x:%08RX64 - SS has both L & D set -> #GP(SS).\n",
2437	uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp));
2438	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
2439	}
2440
2441	/* DPL/RPL/CPL checks. */
2442	if (DescSs.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
2443	{
2444	Log(("retf %04x:%08RX64 %04x:%08RX64 - SS.DPL(%u) != CS.RPL (%u) -> #GP(SS).\n",
2445	uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp, DescSs.Legacy.Gen.u2Dpl, uNewCs & X86_SEL_RPL));
2446	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
2447	}
2448
2449	/* Is it there? */
2450	if (!DescSs.Legacy.Gen.u1Present)
2451	{
2452	Log(("retf %04x:%08RX64 %04x:%08RX64 - SS not present -> #NP(SS).\n", uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp));
2453	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
2454	}
2455
2456	/* Calc SS limit.*/
2457	uint32_t cbLimitSs = X86DESC_LIMIT_G(&DescSs.Legacy);
2458
2459	/* Is RIP canonical or within CS.limit? */
2460	uint64_t u64Base;
2461	uint32_t cbLimitCs = X86DESC_LIMIT_G(&DescCs.Legacy);
2462
2463	/** @todo Testcase: Is this correct? */
2464	if ( DescCs.Legacy.Gen.u1Long
2465	&& IEM_IS_LONG_MODE(pVCpu) )
2466	{
2467	if (!IEM_IS_CANONICAL(uNewRip))
2468	{
2469	Log(("retf %04x:%08RX64 %04x:%08RX64 - not canonical -> #GP.\n", uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp));
2470	return iemRaiseNotCanonical(pVCpu);
2471	}
2472	u64Base = 0;
2473	}
2474	else
2475	{
2476	if (uNewRip > cbLimitCs)
2477	{
2478	Log(("retf %04x:%08RX64 %04x:%08RX64 - out of bounds (%#x)-> #GP(CS).\n",
2479	uNewCs, uNewRip, uNewOuterSs, uNewOuterRsp, cbLimitCs));
2480	/** @todo Intel says this is \#GP(0)! */
2481	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2482	}
2483	u64Base = X86DESC_BASE(&DescCs.Legacy);
2484	}
2485
2486	/*
2487	* Now set the accessed bit before
2488	* writing the return address to the stack and committing the result into
2489	* CS, CSHID and RIP.
2490	*/
2491	/** @todo Testcase: Need to check WHEN exactly the CS accessed bit is set. */
2492	if (!(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2493	{
2494	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
2495	if (rcStrict != VINF_SUCCESS)
2496	return rcStrict;
2497	/** @todo check what VT-x and AMD-V does. */
2498	DescCs.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2499	}
2500	/** @todo Testcase: Need to check WHEN exactly the SS accessed bit is set. */
2501	if (!(DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2502	{
2503	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewOuterSs);
2504	if (rcStrict != VINF_SUCCESS)
2505	return rcStrict;
2506	/** @todo check what VT-x and AMD-V does. */
2507	DescSs.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2508	}
2509
2510	/* commit */
2511	if (enmEffOpSize == IEMMODE_16BIT)
2512	pVCpu->cpum.GstCtx.rip = uNewRip & UINT16_MAX; /** @todo Testcase: When exactly does this occur? With call it happens prior to the limit check according to Intel... */
2513	else
2514	pVCpu->cpum.GstCtx.rip = uNewRip;
2515	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
2516	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
2517	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2518	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCs.Legacy);
2519	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCs;
2520	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
2521	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
2522	pVCpu->cpum.GstCtx.ss.Sel = uNewOuterSs;
2523	pVCpu->cpum.GstCtx.ss.ValidSel = uNewOuterSs;
2524	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
2525	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSs.Legacy);
2526	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSs;
2527	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2528	pVCpu->cpum.GstCtx.ss.u64Base = 0;
2529	else
2530	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSs.Legacy);
2531	if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2532	pVCpu->cpum.GstCtx.sp = (uint16_t)uNewOuterRsp;
2533	else
2534	pVCpu->cpum.GstCtx.rsp = uNewOuterRsp;
2535
2536	pVCpu->iem.s.uCpl = (uNewCs & X86_SEL_RPL);
2537	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.ds);
2538	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.es);
2539	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.fs);
2540	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.gs);
2541
2542	/** @todo check if the hidden bits are loaded correctly for 64-bit
2543	* mode. */
2544
2545	if (cbPop)
2546	iemRegAddToRsp(pVCpu, cbPop);
2547	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
2548
2549	/* Done! */
2550	}
2551	/*
2552	* Return to the same privilege level
2553	*/
2554	else
2555	{
2556	/* Limit / canonical check. */
2557	uint64_t u64Base;
2558	uint32_t cbLimitCs = X86DESC_LIMIT_G(&DescCs.Legacy);
2559
2560	/** @todo Testcase: Is this correct? */
2561	if ( DescCs.Legacy.Gen.u1Long
2562	&& IEM_IS_LONG_MODE(pVCpu) )
2563	{
2564	if (!IEM_IS_CANONICAL(uNewRip))
2565	{
2566	Log(("retf %04x:%08RX64 - not canonical -> #GP\n", uNewCs, uNewRip));
2567	return iemRaiseNotCanonical(pVCpu);
2568	}
2569	u64Base = 0;
2570	}
2571	else
2572	{
2573	if (uNewRip > cbLimitCs)
2574	{
2575	Log(("retf %04x:%08RX64 -> out of bounds (%#x)\n", uNewCs, uNewRip, cbLimitCs));
2576	/** @todo Intel says this is \#GP(0)! */
2577	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2578	}
2579	u64Base = X86DESC_BASE(&DescCs.Legacy);
2580	}
2581
2582	/*
2583	* Now set the accessed bit before
2584	* writing the return address to the stack and committing the result into
2585	* CS, CSHID and RIP.
2586	*/
2587	/** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
2588	if (!(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2589	{
2590	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
2591	if (rcStrict != VINF_SUCCESS)
2592	return rcStrict;
2593	/** @todo check what VT-x and AMD-V does. */
2594	DescCs.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2595	}
2596
2597	/* commit */
2598	if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2599	pVCpu->cpum.GstCtx.sp = (uint16_t)uNewRsp;
2600	else
2601	pVCpu->cpum.GstCtx.rsp = uNewRsp;
2602	if (enmEffOpSize == IEMMODE_16BIT)
2603	pVCpu->cpum.GstCtx.rip = uNewRip & UINT16_MAX; /** @todo Testcase: When exactly does this occur? With call it happens prior to the limit check according to Intel... */
2604	else
2605	pVCpu->cpum.GstCtx.rip = uNewRip;
2606	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
2607	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
2608	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2609	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCs.Legacy);
2610	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCs;
2611	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
2612	/** @todo check if the hidden bits are loaded correctly for 64-bit
2613	* mode. */
2614	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
2615	if (cbPop)
2616	iemRegAddToRsp(pVCpu, cbPop);
2617	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
2618	}
2619
2620	/* Flush the prefetch buffer. */
2621	#ifdef IEM_WITH_CODE_TLB
2622	pVCpu->iem.s.pbInstrBuf = NULL;
2623	#else
2624	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
2625	#endif
2626	return VINF_SUCCESS;
2627	}
2628
2629
2630	/**
2631	* Implements retn.
2632	*
2633	* We're doing this in C because of the \#GP that might be raised if the popped
2634	* program counter is out of bounds.
2635	*
2636	* @param enmEffOpSize The effective operand size.
2637	* @param cbPop The amount of arguments to pop from the stack
2638	* (bytes).
2639	*/
2640	IEM_CIMPL_DEF_2(iemCImpl_retn, IEMMODE, enmEffOpSize, uint16_t, cbPop)
2641	{
2642	NOREF(cbInstr);
2643
2644	/* Fetch the RSP from the stack. */
2645	VBOXSTRICTRC rcStrict;
2646	RTUINT64U NewRip;
2647	RTUINT64U NewRsp;
2648	NewRsp.u = pVCpu->cpum.GstCtx.rsp;
2649
2650	switch (enmEffOpSize)
2651	{
2652	case IEMMODE_16BIT:
2653	NewRip.u = 0;
2654	rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRip.Words.w0, &NewRsp);
2655	break;
2656	case IEMMODE_32BIT:
2657	NewRip.u = 0;
2658	rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRip.DWords.dw0, &NewRsp);
2659	break;
2660	case IEMMODE_64BIT:
2661	rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRip.u, &NewRsp);
2662	break;
2663	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2664	}
2665	if (rcStrict != VINF_SUCCESS)
2666	return rcStrict;
2667
2668	/* Check the new RSP before loading it. */
2669	/** @todo Should test this as the intel+amd pseudo code doesn't mention half
2670	* of it. The canonical test is performed here and for call. */
2671	if (enmEffOpSize != IEMMODE_64BIT)
2672	{
2673	if (NewRip.DWords.dw0 > pVCpu->cpum.GstCtx.cs.u32Limit)
2674	{
2675	Log(("retn newrip=%llx - out of bounds (%x) -> #GP\n", NewRip.u, pVCpu->cpum.GstCtx.cs.u32Limit));
2676	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
2677	}
2678	}
2679	else
2680	{
2681	if (!IEM_IS_CANONICAL(NewRip.u))
2682	{
2683	Log(("retn newrip=%llx - not canonical -> #GP\n", NewRip.u));
2684	return iemRaiseNotCanonical(pVCpu);
2685	}
2686	}
2687
2688	/* Apply cbPop */
2689	if (cbPop)
2690	iemRegAddToRspEx(pVCpu, &NewRsp, cbPop);
2691
2692	/* Commit it. */
2693	pVCpu->cpum.GstCtx.rip = NewRip.u;
2694	pVCpu->cpum.GstCtx.rsp = NewRsp.u;
2695	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
2696
2697	/* Flush the prefetch buffer. */
2698	#ifndef IEM_WITH_CODE_TLB
2699	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
2700	#endif
2701
2702	return VINF_SUCCESS;
2703	}
2704
2705
2706	/**
2707	* Implements enter.
2708	*
2709	* We're doing this in C because the instruction is insane, even for the
2710	* u8NestingLevel=0 case dealing with the stack is tedious.
2711	*
2712	* @param enmEffOpSize The effective operand size.
2713	* @param cbFrame Frame size.
2714	* @param cParameters Frame parameter count.
2715	*/
2716	IEM_CIMPL_DEF_3(iemCImpl_enter, IEMMODE, enmEffOpSize, uint16_t, cbFrame, uint8_t, cParameters)
2717	{
2718	/* Push RBP, saving the old value in TmpRbp. */
2719	RTUINT64U NewRsp; NewRsp.u = pVCpu->cpum.GstCtx.rsp;
2720	RTUINT64U TmpRbp; TmpRbp.u = pVCpu->cpum.GstCtx.rbp;
2721	RTUINT64U NewRbp;
2722	VBOXSTRICTRC rcStrict;
2723	if (enmEffOpSize == IEMMODE_64BIT)
2724	{
2725	rcStrict = iemMemStackPushU64Ex(pVCpu, TmpRbp.u, &NewRsp);
2726	NewRbp = NewRsp;
2727	}
2728	else if (enmEffOpSize == IEMMODE_32BIT)
2729	{
2730	rcStrict = iemMemStackPushU32Ex(pVCpu, TmpRbp.DWords.dw0, &NewRsp);
2731	NewRbp = NewRsp;
2732	}
2733	else
2734	{
2735	rcStrict = iemMemStackPushU16Ex(pVCpu, TmpRbp.Words.w0, &NewRsp);
2736	NewRbp = TmpRbp;
2737	NewRbp.Words.w0 = NewRsp.Words.w0;
2738	}
2739	if (rcStrict != VINF_SUCCESS)
2740	return rcStrict;
2741
2742	/* Copy the parameters (aka nesting levels by Intel). */
2743	cParameters &= 0x1f;
2744	if (cParameters > 0)
2745	{
2746	switch (enmEffOpSize)
2747	{
2748	case IEMMODE_16BIT:
2749	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2750	TmpRbp.DWords.dw0 -= 2;
2751	else
2752	TmpRbp.Words.w0 -= 2;
2753	do
2754	{
2755	uint16_t u16Tmp;
2756	rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Tmp, &TmpRbp);
2757	if (rcStrict != VINF_SUCCESS)
2758	break;
2759	rcStrict = iemMemStackPushU16Ex(pVCpu, u16Tmp, &NewRsp);
2760	} while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
2761	break;
2762
2763	case IEMMODE_32BIT:
2764	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2765	TmpRbp.DWords.dw0 -= 4;
2766	else
2767	TmpRbp.Words.w0 -= 4;
2768	do
2769	{
2770	uint32_t u32Tmp;
2771	rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Tmp, &TmpRbp);
2772	if (rcStrict != VINF_SUCCESS)
2773	break;
2774	rcStrict = iemMemStackPushU32Ex(pVCpu, u32Tmp, &NewRsp);
2775	} while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
2776	break;
2777
2778	case IEMMODE_64BIT:
2779	TmpRbp.u -= 8;
2780	do
2781	{
2782	uint64_t u64Tmp;
2783	rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Tmp, &TmpRbp);
2784	if (rcStrict != VINF_SUCCESS)
2785	break;
2786	rcStrict = iemMemStackPushU64Ex(pVCpu, u64Tmp, &NewRsp);
2787	} while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
2788	break;
2789
2790	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2791	}
2792	if (rcStrict != VINF_SUCCESS)
2793	return VINF_SUCCESS;
2794
2795	/* Push the new RBP */
2796	if (enmEffOpSize == IEMMODE_64BIT)
2797	rcStrict = iemMemStackPushU64Ex(pVCpu, NewRbp.u, &NewRsp);
2798	else if (enmEffOpSize == IEMMODE_32BIT)
2799	rcStrict = iemMemStackPushU32Ex(pVCpu, NewRbp.DWords.dw0, &NewRsp);
2800	else
2801	rcStrict = iemMemStackPushU16Ex(pVCpu, NewRbp.Words.w0, &NewRsp);
2802	if (rcStrict != VINF_SUCCESS)
2803	return rcStrict;
2804
2805	}
2806
2807	/* Recalc RSP. */
2808	iemRegSubFromRspEx(pVCpu, &NewRsp, cbFrame);
2809
2810	/** @todo Should probe write access at the new RSP according to AMD. */
2811	/** @todo Should handle accesses to the VMX APIC-access page. */
2812
2813	/* Commit it. */
2814	pVCpu->cpum.GstCtx.rbp = NewRbp.u;
2815	pVCpu->cpum.GstCtx.rsp = NewRsp.u;
2816	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
2817	}
2818
2819
2820
2821	/**
2822	* Implements leave.
2823	*
2824	* We're doing this in C because messing with the stack registers is annoying
2825	* since they depends on SS attributes.
2826	*
2827	* @param enmEffOpSize The effective operand size.
2828	*/
2829	IEM_CIMPL_DEF_1(iemCImpl_leave, IEMMODE, enmEffOpSize)
2830	{
2831	/* Calculate the intermediate RSP from RBP and the stack attributes. */
2832	RTUINT64U NewRsp;
2833	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2834	NewRsp.u = pVCpu->cpum.GstCtx.rbp;
2835	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2836	NewRsp.u = pVCpu->cpum.GstCtx.ebp;
2837	else
2838	{
2839	/** @todo Check that LEAVE actually preserve the high EBP bits. */
2840	NewRsp.u = pVCpu->cpum.GstCtx.rsp;
2841	NewRsp.Words.w0 = pVCpu->cpum.GstCtx.bp;
2842	}
2843
2844	/* Pop RBP according to the operand size. */
2845	VBOXSTRICTRC rcStrict;
2846	RTUINT64U NewRbp;
2847	switch (enmEffOpSize)
2848	{
2849	case IEMMODE_16BIT:
2850	NewRbp.u = pVCpu->cpum.GstCtx.rbp;
2851	rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRbp.Words.w0, &NewRsp);
2852	break;
2853	case IEMMODE_32BIT:
2854	NewRbp.u = 0;
2855	rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRbp.DWords.dw0, &NewRsp);
2856	break;
2857	case IEMMODE_64BIT:
2858	rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRbp.u, &NewRsp);
2859	break;
2860	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2861	}
2862	if (rcStrict != VINF_SUCCESS)
2863	return rcStrict;
2864
2865
2866	/* Commit it. */
2867	pVCpu->cpum.GstCtx.rbp = NewRbp.u;
2868	pVCpu->cpum.GstCtx.rsp = NewRsp.u;
2869	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
2870	}
2871
2872
2873	/**
2874	* Implements int3 and int XX.
2875	*
2876	* @param u8Int The interrupt vector number.
2877	* @param enmInt The int instruction type.
2878	*/
2879	IEM_CIMPL_DEF_2(iemCImpl_int, uint8_t, u8Int, IEMINT, enmInt)
2880	{
2881	Assert(pVCpu->iem.s.cXcptRecursions == 0);
2882
2883	/*
2884	* We must check if this INT3 might belong to DBGF before raising a #BP.
2885	*/
2886	if (u8Int == 3)
2887	{
2888	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2889	if (pVM->dbgf.ro.cEnabledInt3Breakpoints == 0)
2890	{ /* likely: No vbox debugger breakpoints */ }
2891	else
2892	{
2893	VBOXSTRICTRC rcStrict = DBGFTrap03Handler(pVM, pVCpu, &pVCpu->cpum.GstCtx);
2894	Log(("iemCImpl_int: DBGFTrap03Handler -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
2895	if (rcStrict != VINF_EM_RAW_GUEST_TRAP)
2896	return iemSetPassUpStatus(pVCpu, rcStrict);
2897	}
2898	}
2899	return iemRaiseXcptOrInt(pVCpu,
2900	cbInstr,
2901	u8Int,
2902	IEM_XCPT_FLAGS_T_SOFT_INT \| enmInt,
2903	0,
2904	0);
2905	}
2906
2907
2908	/**
2909	* Implements iret for real mode and V8086 mode.
2910	*
2911	* @param enmEffOpSize The effective operand size.
2912	*/
2913	IEM_CIMPL_DEF_1(iemCImpl_iret_real_v8086, IEMMODE, enmEffOpSize)
2914	{
2915	X86EFLAGS Efl;
2916	Efl.u = IEMMISC_GET_EFL(pVCpu);
2917	NOREF(cbInstr);
2918
2919	/*
2920	* iret throws an exception if VME isn't enabled.
2921	*/
2922	if ( Efl.Bits.u1VM
2923	&& Efl.Bits.u2IOPL != 3
2924	&& !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME))
2925	return iemRaiseGeneralProtectionFault0(pVCpu);
2926
2927	/*
2928	* Do the stack bits, but don't commit RSP before everything checks
2929	* out right.
2930	*/
2931	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2932	VBOXSTRICTRC rcStrict;
2933	RTCPTRUNION uFrame;
2934	uint16_t uNewCs;
2935	uint32_t uNewEip;
2936	uint32_t uNewFlags;
2937	uint64_t uNewRsp;
2938	if (enmEffOpSize == IEMMODE_32BIT)
2939	{
2940	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 12, 1, &uFrame.pv, &uNewRsp);
2941	if (rcStrict != VINF_SUCCESS)
2942	return rcStrict;
2943	uNewEip = uFrame.pu32[0];
2944	if (uNewEip > UINT16_MAX)
2945	return iemRaiseGeneralProtectionFault0(pVCpu);
2946
2947	uNewCs = (uint16_t)uFrame.pu32[1];
2948	uNewFlags = uFrame.pu32[2];
2949	uNewFlags &= X86_EFL_CF \| X86_EFL_PF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF
2950	\| X86_EFL_TF \| X86_EFL_IF \| X86_EFL_DF \| X86_EFL_OF \| X86_EFL_IOPL \| X86_EFL_NT
2951	\| X86_EFL_RF /\| X86_EFL_VM/ \| X86_EFL_AC /\|X86_EFL_VIF/ /\|X86_EFL_VIP/
2952	\| X86_EFL_ID;
2953	if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
2954	uNewFlags &= ~(X86_EFL_AC \| X86_EFL_ID \| X86_EFL_VIF \| X86_EFL_VIP);
2955	uNewFlags \|= Efl.u & (X86_EFL_VM \| X86_EFL_VIF \| X86_EFL_VIP \| X86_EFL_1);
2956	}
2957	else
2958	{
2959	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 6, 1, &uFrame.pv, &uNewRsp);
2960	if (rcStrict != VINF_SUCCESS)
2961	return rcStrict;
2962	uNewEip = uFrame.pu16[0];
2963	uNewCs = uFrame.pu16[1];
2964	uNewFlags = uFrame.pu16[2];
2965	uNewFlags &= X86_EFL_CF \| X86_EFL_PF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF
2966	\| X86_EFL_TF \| X86_EFL_IF \| X86_EFL_DF \| X86_EFL_OF \| X86_EFL_IOPL \| X86_EFL_NT;
2967	uNewFlags \|= Efl.u & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF);
2968	/** @todo The intel pseudo code does not indicate what happens to
2969	* reserved flags. We just ignore them. */
2970	/* Ancient CPU adjustments: See iemCImpl_popf. */
2971	if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_286)
2972	uNewFlags &= ~(X86_EFL_NT \| X86_EFL_IOPL);
2973	}
2974	rcStrict = iemMemStackPopDoneSpecial(pVCpu, uFrame.pv);
2975	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2976	{ /* extremely likely */ }
2977	else
2978	return rcStrict;
2979
2980	/** @todo Check how this is supposed to work if sp=0xfffe. */
2981	Log7(("iemCImpl_iret_real_v8086: uNewCs=%#06x uNewRip=%#010x uNewFlags=%#x uNewRsp=%#18llx\n",
2982	uNewCs, uNewEip, uNewFlags, uNewRsp));
2983
2984	/*
2985	* Check the limit of the new EIP.
2986	*/
2987	/** @todo Only the AMD pseudo code check the limit here, what's
2988	* right? */
2989	if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
2990	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
2991
2992	/*
2993	* V8086 checks and flag adjustments
2994	*/
2995	if (Efl.Bits.u1VM)
2996	{
2997	if (Efl.Bits.u2IOPL == 3)
2998	{
2999	/* Preserve IOPL and clear RF. */
3000	uNewFlags &= ~(X86_EFL_IOPL \| X86_EFL_RF);
3001	uNewFlags \|= Efl.u & (X86_EFL_IOPL);
3002	}
3003	else if ( enmEffOpSize == IEMMODE_16BIT
3004	&& ( !(uNewFlags & X86_EFL_IF)
3005	\|\| !Efl.Bits.u1VIP )
3006	&& !(uNewFlags & X86_EFL_TF) )
3007	{
3008	/* Move IF to VIF, clear RF and preserve IF and IOPL.*/
3009	uNewFlags &= ~X86_EFL_VIF;
3010	uNewFlags \|= (uNewFlags & X86_EFL_IF) << (19 - 9);
3011	uNewFlags &= ~(X86_EFL_IF \| X86_EFL_IOPL \| X86_EFL_RF);
3012	uNewFlags \|= Efl.u & (X86_EFL_IF \| X86_EFL_IOPL);
3013	}
3014	else
3015	return iemRaiseGeneralProtectionFault0(pVCpu);
3016	Log7(("iemCImpl_iret_real_v8086: u1VM=1: adjusted uNewFlags=%#x\n", uNewFlags));
3017	}
3018
3019	/*
3020	* Commit the operation.
3021	*/
3022	#ifdef DBGFTRACE_ENABLED
3023	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/rm %04x:%04x -> %04x:%04x %x %04llx",
3024	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewRsp);
3025	#endif
3026	pVCpu->cpum.GstCtx.rsp = uNewRsp;
3027	pVCpu->cpum.GstCtx.rip = uNewEip;
3028	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
3029	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
3030	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3031	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uNewCs << 4;
3032	/** @todo do we load attribs and limit as well? */
3033	Assert(uNewFlags & X86_EFL_1);
3034	IEMMISC_SET_EFL(pVCpu, uNewFlags);
3035
3036	/* Flush the prefetch buffer. */
3037	#ifdef IEM_WITH_CODE_TLB
3038	pVCpu->iem.s.pbInstrBuf = NULL;
3039	#else
3040	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
3041	#endif
3042
3043	return VINF_SUCCESS;
3044	}
3045
3046
3047	/**
3048	* Loads a segment register when entering V8086 mode.
3049	*
3050	* @param pSReg The segment register.
3051	* @param uSeg The segment to load.
3052	*/
3053	static void iemCImplCommonV8086LoadSeg(PCPUMSELREG pSReg, uint16_t uSeg)
3054	{
3055	pSReg->Sel = uSeg;
3056	pSReg->ValidSel = uSeg;
3057	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3058	pSReg->u64Base = (uint32_t)uSeg << 4;
3059	pSReg->u32Limit = 0xffff;
3060	pSReg->Attr.u = X86_SEL_TYPE_RW_ACC \| RT_BIT(4) /!sys/ \| RT_BIT(7) /P/ \| (3 /DPL/ << 5); /* VT-x wants 0xf3 */
3061	/** @todo Testcase: Check if VT-x really needs this and what it does itself when
3062	* IRET'ing to V8086. */
3063	}
3064
3065
3066	/**
3067	* Implements iret for protected mode returning to V8086 mode.
3068	*
3069	* @param uNewEip The new EIP.
3070	* @param uNewCs The new CS.
3071	* @param uNewFlags The new EFLAGS.
3072	* @param uNewRsp The RSP after the initial IRET frame.
3073	*
3074	* @note This can only be a 32-bit iret du to the X86_EFL_VM position.
3075	*/
3076	IEM_CIMPL_DEF_4(iemCImpl_iret_prot_v8086, uint32_t, uNewEip, uint16_t, uNewCs, uint32_t, uNewFlags, uint64_t, uNewRsp)
3077	{
3078	RT_NOREF_PV(cbInstr);
3079	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
3080
3081	/*
3082	* Pop the V8086 specific frame bits off the stack.
3083	*/
3084	VBOXSTRICTRC rcStrict;
3085	RTCPTRUNION uFrame;
3086	rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0 /off/, 24 /cbMem/, &uFrame.pv, uNewRsp);
3087	if (rcStrict != VINF_SUCCESS)
3088	return rcStrict;
3089	uint32_t uNewEsp = uFrame.pu32[0];
3090	uint16_t uNewSs = uFrame.pu32[1];
3091	uint16_t uNewEs = uFrame.pu32[2];
3092	uint16_t uNewDs = uFrame.pu32[3];
3093	uint16_t uNewFs = uFrame.pu32[4];
3094	uint16_t uNewGs = uFrame.pu32[5];
3095	rcStrict = iemMemCommitAndUnmap(pVCpu, (void )uFrame.pv, IEM_ACCESS_STACK_R); / don't use iemMemStackPopCommitSpecial here. */
3096	if (rcStrict != VINF_SUCCESS)
3097	return rcStrict;
3098
3099	/*
3100	* Commit the operation.
3101	*/
3102	uNewFlags &= X86_EFL_LIVE_MASK;
3103	uNewFlags \|= X86_EFL_RA1_MASK;
3104	#ifdef DBGFTRACE_ENABLED
3105	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/p/v %04x:%08x -> %04x:%04x %x %04x:%04x",
3106	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewSs, uNewEsp);
3107	#endif
3108	Log7(("iemCImpl_iret_prot_v8086: %04x:%08x -> %04x:%04x %x %04x:%04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewSs, uNewEsp));
3109
3110	IEMMISC_SET_EFL(pVCpu, uNewFlags);
3111	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.cs, uNewCs);
3112	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.ss, uNewSs);
3113	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.es, uNewEs);
3114	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.ds, uNewDs);
3115	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.fs, uNewFs);
3116	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.gs, uNewGs);
3117	pVCpu->cpum.GstCtx.rip = (uint16_t)uNewEip;
3118	pVCpu->cpum.GstCtx.rsp = uNewEsp; /** @todo check this out! */
3119	pVCpu->iem.s.uCpl = 3;
3120
3121	/* Flush the prefetch buffer. */
3122	#ifdef IEM_WITH_CODE_TLB
3123	pVCpu->iem.s.pbInstrBuf = NULL;
3124	#else
3125	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
3126	#endif
3127
3128	return VINF_SUCCESS;
3129	}
3130
3131
3132	/**
3133	* Implements iret for protected mode returning via a nested task.
3134	*
3135	* @param enmEffOpSize The effective operand size.
3136	*/
3137	IEM_CIMPL_DEF_1(iemCImpl_iret_prot_NestedTask, IEMMODE, enmEffOpSize)
3138	{
3139	Log7(("iemCImpl_iret_prot_NestedTask:\n"));
3140	#ifndef IEM_IMPLEMENTS_TASKSWITCH
3141	IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
3142	#else
3143	RT_NOREF_PV(enmEffOpSize);
3144
3145	/*
3146	* Read the segment selector in the link-field of the current TSS.
3147	*/
3148	RTSEL uSelRet;
3149	VBOXSTRICTRC rcStrict = iemMemFetchSysU16(pVCpu, &uSelRet, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base);
3150	if (rcStrict != VINF_SUCCESS)
3151	return rcStrict;
3152
3153	/*
3154	* Fetch the returning task's TSS descriptor from the GDT.
3155	*/
3156	if (uSelRet & X86_SEL_LDT)
3157	{
3158	Log(("iret_prot_NestedTask TSS not in LDT. uSelRet=%04x -> #TS\n", uSelRet));
3159	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet);
3160	}
3161
3162	IEMSELDESC TssDesc;
3163	rcStrict = iemMemFetchSelDesc(pVCpu, &TssDesc, uSelRet, X86_XCPT_GP);
3164	if (rcStrict != VINF_SUCCESS)
3165	return rcStrict;
3166
3167	if (TssDesc.Legacy.Gate.u1DescType)
3168	{
3169	Log(("iret_prot_NestedTask Invalid TSS type. uSelRet=%04x -> #TS\n", uSelRet));
3170	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
3171	}
3172
3173	if ( TssDesc.Legacy.Gate.u4Type != X86_SEL_TYPE_SYS_286_TSS_BUSY
3174	&& TssDesc.Legacy.Gate.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
3175	{
3176	Log(("iret_prot_NestedTask TSS is not busy. uSelRet=%04x DescType=%#x -> #TS\n", uSelRet, TssDesc.Legacy.Gate.u4Type));
3177	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
3178	}
3179
3180	if (!TssDesc.Legacy.Gate.u1Present)
3181	{
3182	Log(("iret_prot_NestedTask TSS is not present. uSelRet=%04x -> #NP\n", uSelRet));
3183	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
3184	}
3185
3186	uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
3187	return iemTaskSwitch(pVCpu, IEMTASKSWITCH_IRET, uNextEip, 0 /* fFlags /, 0 / uErr */,
3188	0 /* uCr2 */, uSelRet, &TssDesc);
3189	#endif
3190	}
3191
3192
3193	/**
3194	* Implements iret for protected mode
3195	*
3196	* @param enmEffOpSize The effective operand size.
3197	*/
3198	IEM_CIMPL_DEF_1(iemCImpl_iret_prot, IEMMODE, enmEffOpSize)
3199	{
3200	NOREF(cbInstr);
3201	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
3202
3203	/*
3204	* Nested task return.
3205	*/
3206	if (pVCpu->cpum.GstCtx.eflags.Bits.u1NT)
3207	return IEM_CIMPL_CALL_1(iemCImpl_iret_prot_NestedTask, enmEffOpSize);
3208
3209	/*
3210	* Normal return.
3211	*
3212	* Do the stack bits, but don't commit RSP before everything checks
3213	* out right.
3214	*/
3215	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
3216	VBOXSTRICTRC rcStrict;
3217	RTCPTRUNION uFrame;
3218	uint16_t uNewCs;
3219	uint32_t uNewEip;
3220	uint32_t uNewFlags;
3221	uint64_t uNewRsp;
3222	if (enmEffOpSize == IEMMODE_32BIT)
3223	{
3224	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 12, 3, &uFrame.pv, &uNewRsp);
3225	if (rcStrict != VINF_SUCCESS)
3226	return rcStrict;
3227	uNewEip = uFrame.pu32[0];
3228	uNewCs = (uint16_t)uFrame.pu32[1];
3229	uNewFlags = uFrame.pu32[2];
3230	}
3231	else
3232	{
3233	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 6, 1, &uFrame.pv, &uNewRsp);
3234	if (rcStrict != VINF_SUCCESS)
3235	return rcStrict;
3236	uNewEip = uFrame.pu16[0];
3237	uNewCs = uFrame.pu16[1];
3238	uNewFlags = uFrame.pu16[2];
3239	}
3240	rcStrict = iemMemStackPopDoneSpecial(pVCpu, (void )uFrame.pv); / don't use iemMemStackPopCommitSpecial here. */
3241	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3242	{ /* extremely likely */ }
3243	else
3244	return rcStrict;
3245	Log7(("iemCImpl_iret_prot: uNewCs=%#06x uNewEip=%#010x uNewFlags=%#x uNewRsp=%#18llx uCpl=%u\n", uNewCs, uNewEip, uNewFlags, uNewRsp, pVCpu->iem.s.uCpl));
3246
3247	/*
3248	* We're hopefully not returning to V8086 mode...
3249	*/
3250	if ( (uNewFlags & X86_EFL_VM)
3251	&& pVCpu->iem.s.uCpl == 0)
3252	{
3253	Assert(enmEffOpSize == IEMMODE_32BIT);
3254	return IEM_CIMPL_CALL_4(iemCImpl_iret_prot_v8086, uNewEip, uNewCs, uNewFlags, uNewRsp);
3255	}
3256
3257	/*
3258	* Protected mode.
3259	*/
3260	/* Read the CS descriptor. */
3261	if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
3262	{
3263	Log(("iret %04x:%08x -> invalid CS selector, #GP(0)\n", uNewCs, uNewEip));
3264	return iemRaiseGeneralProtectionFault0(pVCpu);
3265	}
3266
3267	IEMSELDESC DescCS;
3268	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCs, X86_XCPT_GP);
3269	if (rcStrict != VINF_SUCCESS)
3270	{
3271	Log(("iret %04x:%08x - rcStrict=%Rrc when fetching CS\n", uNewCs, uNewEip, VBOXSTRICTRC_VAL(rcStrict)));
3272	return rcStrict;
3273	}
3274
3275	/* Must be a code descriptor. */
3276	if (!DescCS.Legacy.Gen.u1DescType)
3277	{
3278	Log(("iret %04x:%08x - CS is system segment (%#x) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u4Type));
3279	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3280	}
3281	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3282	{
3283	Log(("iret %04x:%08x - not code segment (%#x) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u4Type));
3284	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3285	}
3286
3287	/* Privilege checks. */
3288	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF))
3289	{
3290	if ((uNewCs & X86_SEL_RPL) != DescCS.Legacy.Gen.u2Dpl)
3291	{
3292	Log(("iret %04x:%08x - RPL != DPL (%d) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u2Dpl));
3293	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3294	}
3295	}
3296	else if ((uNewCs & X86_SEL_RPL) < DescCS.Legacy.Gen.u2Dpl)
3297	{
3298	Log(("iret %04x:%08x - RPL < DPL (%d) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u2Dpl));
3299	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3300	}
3301	if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl)
3302	{
3303	Log(("iret %04x:%08x - RPL < CPL (%d) -> #GP\n", uNewCs, uNewEip, pVCpu->iem.s.uCpl));
3304	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3305	}
3306
3307	/* Present? */
3308	if (!DescCS.Legacy.Gen.u1Present)
3309	{
3310	Log(("iret %04x:%08x - CS not present -> #NP\n", uNewCs, uNewEip));
3311	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
3312	}
3313
3314	uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
3315
3316	/*
3317	* Return to outer level?
3318	*/
3319	if ((uNewCs & X86_SEL_RPL) != pVCpu->iem.s.uCpl)
3320	{
3321	uint16_t uNewSS;
3322	uint32_t uNewESP;
3323	if (enmEffOpSize == IEMMODE_32BIT)
3324	{
3325	rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0/off/, 8 /cbMem/, &uFrame.pv, uNewRsp);
3326	if (rcStrict != VINF_SUCCESS)
3327	return rcStrict;
3328	/** @todo We might be popping a 32-bit ESP from the IRET frame, but whether
3329	* 16-bit or 32-bit are being loaded into SP depends on the D/B
3330	* bit of the popped SS selector it turns out. */
3331	uNewESP = uFrame.pu32[0];
3332	uNewSS = (uint16_t)uFrame.pu32[1];
3333	}
3334	else
3335	{
3336	rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0 /off/, 4 /cbMem/, &uFrame.pv, uNewRsp);
3337	if (rcStrict != VINF_SUCCESS)
3338	return rcStrict;
3339	uNewESP = uFrame.pu16[0];
3340	uNewSS = uFrame.pu16[1];
3341	}
3342	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uFrame.pv, IEM_ACCESS_STACK_R);
3343	if (rcStrict != VINF_SUCCESS)
3344	return rcStrict;
3345	Log7(("iemCImpl_iret_prot: uNewSS=%#06x uNewESP=%#010x\n", uNewSS, uNewESP));
3346
3347	/* Read the SS descriptor. */
3348	if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
3349	{
3350	Log(("iret %04x:%08x/%04x:%08x -> invalid SS selector, #GP(0)\n", uNewCs, uNewEip, uNewSS, uNewESP));
3351	return iemRaiseGeneralProtectionFault0(pVCpu);
3352	}
3353
3354	IEMSELDESC DescSS;
3355	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_GP); /** @todo Correct exception? */
3356	if (rcStrict != VINF_SUCCESS)
3357	{
3358	Log(("iret %04x:%08x/%04x:%08x - %Rrc when fetching SS\n",
3359	uNewCs, uNewEip, uNewSS, uNewESP, VBOXSTRICTRC_VAL(rcStrict)));
3360	return rcStrict;
3361	}
3362
3363	/* Privilege checks. */
3364	if ((uNewSS & X86_SEL_RPL) != (uNewCs & X86_SEL_RPL))
3365	{
3366	Log(("iret %04x:%08x/%04x:%08x -> SS.RPL != CS.RPL -> #GP\n", uNewCs, uNewEip, uNewSS, uNewESP));
3367	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
3368	}
3369	if (DescSS.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
3370	{
3371	Log(("iret %04x:%08x/%04x:%08x -> SS.DPL (%d) != CS.RPL -> #GP\n",
3372	uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u2Dpl));
3373	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
3374	}
3375
3376	/* Must be a writeable data segment descriptor. */
3377	if (!DescSS.Legacy.Gen.u1DescType)
3378	{
3379	Log(("iret %04x:%08x/%04x:%08x -> SS is system segment (%#x) -> #GP\n",
3380	uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u4Type));
3381	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
3382	}
3383	if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
3384	{
3385	Log(("iret %04x:%08x/%04x:%08x - not writable data segment (%#x) -> #GP\n",
3386	uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u4Type));
3387	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
3388	}
3389
3390	/* Present? */
3391	if (!DescSS.Legacy.Gen.u1Present)
3392	{
3393	Log(("iret %04x:%08x/%04x:%08x -> SS not present -> #SS\n", uNewCs, uNewEip, uNewSS, uNewESP));
3394	return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSS);
3395	}
3396
3397	uint32_t cbLimitSs = X86DESC_LIMIT_G(&DescSS.Legacy);
3398
3399	/* Check EIP. */
3400	if (uNewEip > cbLimitCS)
3401	{
3402	Log(("iret %04x:%08x/%04x:%08x -> EIP is out of bounds (%#x) -> #GP(0)\n",
3403	uNewCs, uNewEip, uNewSS, uNewESP, cbLimitCS));
3404	/** @todo Which is it, \#GP(0) or \#GP(sel)? */
3405	return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
3406	}
3407
3408	/*
3409	* Commit the changes, marking CS and SS accessed first since
3410	* that may fail.
3411	*/
3412	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3413	{
3414	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
3415	if (rcStrict != VINF_SUCCESS)
3416	return rcStrict;
3417	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3418	}
3419	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3420	{
3421	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
3422	if (rcStrict != VINF_SUCCESS)
3423	return rcStrict;
3424	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3425	}
3426
3427	uint32_t fEFlagsMask = X86_EFL_CF \| X86_EFL_PF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF
3428	\| X86_EFL_TF \| X86_EFL_DF \| X86_EFL_OF \| X86_EFL_NT;
3429	if (enmEffOpSize != IEMMODE_16BIT)
3430	fEFlagsMask \|= X86_EFL_RF \| X86_EFL_AC \| X86_EFL_ID;
3431	if (pVCpu->iem.s.uCpl == 0)
3432	fEFlagsMask \|= X86_EFL_IF \| X86_EFL_IOPL \| X86_EFL_VIF \| X86_EFL_VIP; /* VM is 0 */
3433	else if (pVCpu->iem.s.uCpl <= pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL)
3434	fEFlagsMask \|= X86_EFL_IF;
3435	if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
3436	fEFlagsMask &= ~(X86_EFL_AC \| X86_EFL_ID \| X86_EFL_VIF \| X86_EFL_VIP);
3437	uint32_t fEFlagsNew = IEMMISC_GET_EFL(pVCpu);
3438	fEFlagsNew &= ~fEFlagsMask;
3439	fEFlagsNew \|= uNewFlags & fEFlagsMask;
3440	#ifdef DBGFTRACE_ENABLED
3441	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%up%u %04x:%08x -> %04x:%04x %x %04x:%04x",
3442	pVCpu->iem.s.uCpl, uNewCs & X86_SEL_RPL, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
3443	uNewCs, uNewEip, uNewFlags, uNewSS, uNewESP);
3444	#endif
3445
3446	IEMMISC_SET_EFL(pVCpu, fEFlagsNew);
3447	pVCpu->cpum.GstCtx.rip = uNewEip;
3448	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
3449	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
3450	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3451	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3452	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3453	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3454	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
3455
3456	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
3457	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
3458	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3459	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3460	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSs;
3461	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
3462	if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3463	pVCpu->cpum.GstCtx.sp = (uint16_t)uNewESP;
3464	else
3465	pVCpu->cpum.GstCtx.rsp = uNewESP;
3466
3467	pVCpu->iem.s.uCpl = uNewCs & X86_SEL_RPL;
3468	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.ds);
3469	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.es);
3470	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.fs);
3471	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.gs);
3472
3473	/* Done! */
3474
3475	}
3476	/*
3477	* Return to the same level.
3478	*/
3479	else
3480	{
3481	/* Check EIP. */
3482	if (uNewEip > cbLimitCS)
3483	{
3484	Log(("iret %04x:%08x - EIP is out of bounds (%#x) -> #GP(0)\n", uNewCs, uNewEip, cbLimitCS));
3485	/** @todo Which is it, \#GP(0) or \#GP(sel)? */
3486	return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
3487	}
3488
3489	/*
3490	* Commit the changes, marking CS first since it may fail.
3491	*/
3492	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3493	{
3494	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
3495	if (rcStrict != VINF_SUCCESS)
3496	return rcStrict;
3497	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3498	}
3499
3500	X86EFLAGS NewEfl;
3501	NewEfl.u = IEMMISC_GET_EFL(pVCpu);
3502	uint32_t fEFlagsMask = X86_EFL_CF \| X86_EFL_PF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF
3503	\| X86_EFL_TF \| X86_EFL_DF \| X86_EFL_OF \| X86_EFL_NT;
3504	if (enmEffOpSize != IEMMODE_16BIT)
3505	fEFlagsMask \|= X86_EFL_RF \| X86_EFL_AC \| X86_EFL_ID;
3506	if (pVCpu->iem.s.uCpl == 0)
3507	fEFlagsMask \|= X86_EFL_IF \| X86_EFL_IOPL \| X86_EFL_VIF \| X86_EFL_VIP; /* VM is 0 */
3508	else if (pVCpu->iem.s.uCpl <= NewEfl.Bits.u2IOPL)
3509	fEFlagsMask \|= X86_EFL_IF;
3510	if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
3511	fEFlagsMask &= ~(X86_EFL_AC \| X86_EFL_ID \| X86_EFL_VIF \| X86_EFL_VIP);
3512	NewEfl.u &= ~fEFlagsMask;
3513	NewEfl.u \|= fEFlagsMask & uNewFlags;
3514	#ifdef DBGFTRACE_ENABLED
3515	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%up %04x:%08x -> %04x:%04x %x %04x:%04llx",
3516	pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
3517	uNewCs, uNewEip, uNewFlags, pVCpu->cpum.GstCtx.ss.Sel, uNewRsp);
3518	#endif
3519
3520	IEMMISC_SET_EFL(pVCpu, NewEfl.u);
3521	pVCpu->cpum.GstCtx.rip = uNewEip;
3522	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
3523	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
3524	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3525	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3526	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3527	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3528	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
3529	if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3530	pVCpu->cpum.GstCtx.sp = (uint16_t)uNewRsp;
3531	else
3532	pVCpu->cpum.GstCtx.rsp = uNewRsp;
3533	/* Done! */
3534	}
3535
3536	/* Flush the prefetch buffer. */
3537	#ifdef IEM_WITH_CODE_TLB
3538	pVCpu->iem.s.pbInstrBuf = NULL;
3539	#else
3540	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
3541	#endif
3542
3543	return VINF_SUCCESS;
3544	}
3545
3546
3547	/**
3548	* Implements iret for long mode
3549	*
3550	* @param enmEffOpSize The effective operand size.
3551	*/
3552	IEM_CIMPL_DEF_1(iemCImpl_iret_64bit, IEMMODE, enmEffOpSize)
3553	{
3554	NOREF(cbInstr);
3555
3556	/*
3557	* Nested task return is not supported in long mode.
3558	*/
3559	if (pVCpu->cpum.GstCtx.eflags.Bits.u1NT)
3560	{
3561	Log(("iretq with NT=1 (eflags=%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.eflags.u));
3562	return iemRaiseGeneralProtectionFault0(pVCpu);
3563	}
3564
3565	/*
3566	* Normal return.
3567	*
3568	* Do the stack bits, but don't commit RSP before everything checks
3569	* out right.
3570	*/
3571	VBOXSTRICTRC rcStrict;
3572	RTCPTRUNION uFrame;
3573	uint64_t uNewRip;
3574	uint16_t uNewCs;
3575	uint16_t uNewSs;
3576	uint32_t uNewFlags;
3577	uint64_t uNewRsp;
3578	if (enmEffOpSize == IEMMODE_64BIT)
3579	{
3580	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*8, 7, &uFrame.pv, &uNewRsp);
3581	if (rcStrict != VINF_SUCCESS)
3582	return rcStrict;
3583	uNewRip = uFrame.pu64[0];
3584	uNewCs = (uint16_t)uFrame.pu64[1];
3585	uNewFlags = (uint32_t)uFrame.pu64[2];
3586	uNewRsp = uFrame.pu64[3];
3587	uNewSs = (uint16_t)uFrame.pu64[4];
3588	}
3589	else if (enmEffOpSize == IEMMODE_32BIT)
3590	{
3591	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*4, 3, &uFrame.pv, &uNewRsp);
3592	if (rcStrict != VINF_SUCCESS)
3593	return rcStrict;
3594	uNewRip = uFrame.pu32[0];
3595	uNewCs = (uint16_t)uFrame.pu32[1];
3596	uNewFlags = uFrame.pu32[2];
3597	uNewRsp = uFrame.pu32[3];
3598	uNewSs = (uint16_t)uFrame.pu32[4];
3599	}
3600	else
3601	{
3602	Assert(enmEffOpSize == IEMMODE_16BIT);
3603	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*2, 1, &uFrame.pv, &uNewRsp);
3604	if (rcStrict != VINF_SUCCESS)
3605	return rcStrict;
3606	uNewRip = uFrame.pu16[0];
3607	uNewCs = uFrame.pu16[1];
3608	uNewFlags = uFrame.pu16[2];
3609	uNewRsp = uFrame.pu16[3];
3610	uNewSs = uFrame.pu16[4];
3611	}
3612	rcStrict = iemMemStackPopDoneSpecial(pVCpu, (void )uFrame.pv); / don't use iemMemStackPopCommitSpecial here. */
3613	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3614	{ /* extremely like */ }
3615	else
3616	return rcStrict;
3617	Log7(("iretq stack: cs:rip=%04x:%016RX64 rflags=%016RX64 ss:rsp=%04x:%016RX64\n", uNewCs, uNewRip, uNewFlags, uNewSs, uNewRsp));
3618
3619	/*
3620	* Check stuff.
3621	*/
3622	/* Read the CS descriptor. */
3623	if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
3624	{
3625	Log(("iret %04x:%016RX64/%04x:%016RX64 -> invalid CS selector, #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3626	return iemRaiseGeneralProtectionFault0(pVCpu);
3627	}
3628
3629	IEMSELDESC DescCS;
3630	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCs, X86_XCPT_GP);
3631	if (rcStrict != VINF_SUCCESS)
3632	{
3633	Log(("iret %04x:%016RX64/%04x:%016RX64 - rcStrict=%Rrc when fetching CS\n",
3634	uNewCs, uNewRip, uNewSs, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
3635	return rcStrict;
3636	}
3637
3638	/* Must be a code descriptor. */
3639	if ( !DescCS.Legacy.Gen.u1DescType
3640	\|\| !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3641	{
3642	Log(("iret %04x:%016RX64/%04x:%016RX64 - CS is not a code segment T=%u T=%#xu -> #GP\n",
3643	uNewCs, uNewRip, uNewSs, uNewRsp, DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
3644	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3645	}
3646
3647	/* Privilege checks. */
3648	uint8_t const uNewCpl = uNewCs & X86_SEL_RPL;
3649	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF))
3650	{
3651	if ((uNewCs & X86_SEL_RPL) != DescCS.Legacy.Gen.u2Dpl)
3652	{
3653	Log(("iret %04x:%016RX64 - RPL != DPL (%d) -> #GP\n", uNewCs, uNewRip, DescCS.Legacy.Gen.u2Dpl));
3654	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3655	}
3656	}
3657	else if ((uNewCs & X86_SEL_RPL) < DescCS.Legacy.Gen.u2Dpl)
3658	{
3659	Log(("iret %04x:%016RX64 - RPL < DPL (%d) -> #GP\n", uNewCs, uNewRip, DescCS.Legacy.Gen.u2Dpl));
3660	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3661	}
3662	if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl)
3663	{
3664	Log(("iret %04x:%016RX64 - RPL < CPL (%d) -> #GP\n", uNewCs, uNewRip, pVCpu->iem.s.uCpl));
3665	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3666	}
3667
3668	/* Present? */
3669	if (!DescCS.Legacy.Gen.u1Present)
3670	{
3671	Log(("iret %04x:%016RX64/%04x:%016RX64 - CS not present -> #NP\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3672	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
3673	}
3674
3675	uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
3676
3677	/* Read the SS descriptor. */
3678	IEMSELDESC DescSS;
3679	if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
3680	{
3681	if ( !DescCS.Legacy.Gen.u1Long
3682	\|\| DescCS.Legacy.Gen.u1DefBig /** @todo exactly how does iret (and others) behave with u1Long=1 and u1DefBig=1? \#GP(sel)? */
3683	\|\| uNewCpl > 2) /** @todo verify SS=0 impossible for ring-3. */
3684	{
3685	Log(("iret %04x:%016RX64/%04x:%016RX64 -> invalid SS selector, #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3686	return iemRaiseGeneralProtectionFault0(pVCpu);
3687	}
3688	/* Make sure SS is sensible, marked as accessed etc. */
3689	iemMemFakeStackSelDesc(&DescSS, (uNewSs & X86_SEL_RPL));
3690	}
3691	else
3692	{
3693	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSs, X86_XCPT_GP); /** @todo Correct exception? */
3694	if (rcStrict != VINF_SUCCESS)
3695	{
3696	Log(("iret %04x:%016RX64/%04x:%016RX64 - %Rrc when fetching SS\n",
3697	uNewCs, uNewRip, uNewSs, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
3698	return rcStrict;
3699	}
3700	}
3701
3702	/* Privilege checks. */
3703	if ((uNewSs & X86_SEL_RPL) != (uNewCs & X86_SEL_RPL))
3704	{
3705	Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS.RPL != CS.RPL -> #GP\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3706	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
3707	}
3708
3709	uint32_t cbLimitSs;
3710	if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
3711	cbLimitSs = UINT32_MAX;
3712	else
3713	{
3714	if (DescSS.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
3715	{
3716	Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS.DPL (%d) != CS.RPL -> #GP\n",
3717	uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u2Dpl));
3718	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
3719	}
3720
3721	/* Must be a writeable data segment descriptor. */
3722	if (!DescSS.Legacy.Gen.u1DescType)
3723	{
3724	Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS is system segment (%#x) -> #GP\n",
3725	uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u4Type));
3726	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
3727	}
3728	if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
3729	{
3730	Log(("iret %04x:%016RX64/%04x:%016RX64 - not writable data segment (%#x) -> #GP\n",
3731	uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u4Type));
3732	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
3733	}
3734
3735	/* Present? */
3736	if (!DescSS.Legacy.Gen.u1Present)
3737	{
3738	Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS not present -> #SS\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3739	return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSs);
3740	}
3741	cbLimitSs = X86DESC_LIMIT_G(&DescSS.Legacy);
3742	}
3743
3744	/* Check EIP. */
3745	if (DescCS.Legacy.Gen.u1Long)
3746	{
3747	if (!IEM_IS_CANONICAL(uNewRip))
3748	{
3749	Log(("iret %04x:%016RX64/%04x:%016RX64 -> RIP is not canonical -> #GP(0)\n",
3750	uNewCs, uNewRip, uNewSs, uNewRsp));
3751	return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
3752	}
3753	}
3754	else
3755	{
3756	if (uNewRip > cbLimitCS)
3757	{
3758	Log(("iret %04x:%016RX64/%04x:%016RX64 -> EIP is out of bounds (%#x) -> #GP(0)\n",
3759	uNewCs, uNewRip, uNewSs, uNewRsp, cbLimitCS));
3760	/** @todo Which is it, \#GP(0) or \#GP(sel)? */
3761	return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
3762	}
3763	}
3764
3765	/*
3766	* Commit the changes, marking CS and SS accessed first since
3767	* that may fail.
3768	*/
3769	/** @todo where exactly are these actually marked accessed by a real CPU? */
3770	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3771	{
3772	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
3773	if (rcStrict != VINF_SUCCESS)
3774	return rcStrict;
3775	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3776	}
3777	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3778	{
3779	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSs);
3780	if (rcStrict != VINF_SUCCESS)
3781	return rcStrict;
3782	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3783	}
3784
3785	uint32_t fEFlagsMask = X86_EFL_CF \| X86_EFL_PF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF
3786	\| X86_EFL_TF \| X86_EFL_DF \| X86_EFL_OF \| X86_EFL_NT;
3787	if (enmEffOpSize != IEMMODE_16BIT)
3788	fEFlagsMask \|= X86_EFL_RF \| X86_EFL_AC \| X86_EFL_ID;
3789	if (pVCpu->iem.s.uCpl == 0)
3790	fEFlagsMask \|= X86_EFL_IF \| X86_EFL_IOPL \| X86_EFL_VIF \| X86_EFL_VIP; /* VM is ignored */
3791	else if (pVCpu->iem.s.uCpl <= pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL)
3792	fEFlagsMask \|= X86_EFL_IF;
3793	uint32_t fEFlagsNew = IEMMISC_GET_EFL(pVCpu);
3794	fEFlagsNew &= ~fEFlagsMask;
3795	fEFlagsNew \|= uNewFlags & fEFlagsMask;
3796	#ifdef DBGFTRACE_ENABLED
3797	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%ul%u %08llx -> %04x:%04llx %llx %04x:%04llx",
3798	pVCpu->iem.s.uCpl, uNewCpl, pVCpu->cpum.GstCtx.rip, uNewCs, uNewRip, uNewFlags, uNewSs, uNewRsp);
3799	#endif
3800
3801	IEMMISC_SET_EFL(pVCpu, fEFlagsNew);
3802	pVCpu->cpum.GstCtx.rip = uNewRip;
3803	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
3804	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
3805	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3806	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3807	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3808	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3809	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
3810	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1Long \|\| pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
3811	pVCpu->cpum.GstCtx.rsp = uNewRsp;
3812	else
3813	pVCpu->cpum.GstCtx.sp = (uint16_t)uNewRsp;
3814	pVCpu->cpum.GstCtx.ss.Sel = uNewSs;
3815	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSs;
3816	if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
3817	{
3818	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3819	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_UNUSABLE \| (uNewCpl << X86DESCATTR_DPL_SHIFT);
3820	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
3821	pVCpu->cpum.GstCtx.ss.u64Base = 0;
3822	Log2(("iretq new SS: NULL\n"));
3823	}
3824	else
3825	{
3826	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3827	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3828	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSs;
3829	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
3830	Log2(("iretq new SS: base=%#RX64 lim=%#x attr=%#x\n", pVCpu->cpum.GstCtx.ss.u64Base, pVCpu->cpum.GstCtx.ss.u32Limit, pVCpu->cpum.GstCtx.ss.Attr.u));
3831	}
3832
3833	if (pVCpu->iem.s.uCpl != uNewCpl)
3834	{
3835	pVCpu->iem.s.uCpl = uNewCpl;
3836	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.ds);
3837	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.es);
3838	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.fs);
3839	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.gs);
3840	}
3841
3842	/* Flush the prefetch buffer. */
3843	#ifdef IEM_WITH_CODE_TLB
3844	pVCpu->iem.s.pbInstrBuf = NULL;
3845	#else
3846	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
3847	#endif
3848
3849	return VINF_SUCCESS;
3850	}
3851
3852
3853	/**
3854	* Implements iret.
3855	*
3856	* @param enmEffOpSize The effective operand size.
3857	*/
3858	IEM_CIMPL_DEF_1(iemCImpl_iret, IEMMODE, enmEffOpSize)
3859	{
3860	bool fBlockingNmi = CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx);
3861
3862	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3863	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3864	{
3865	/*
3866	* Record whether NMI (or virtual-NMI) blocking is in effect during the execution
3867	* of this IRET instruction. We need to provide this information as part of some
3868	* VM-exits.
3869	*
3870	* See Intel spec. 27.2.2 "Information for VM Exits Due to Vectored Events".
3871	*/
3872	if (IEM_VMX_IS_PINCTLS_SET(pVCpu, VMX_PIN_CTLS_VIRT_NMI))
3873	pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking;
3874	else
3875	pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = fBlockingNmi;
3876
3877	/*
3878	* If "NMI exiting" is set, IRET does not affect blocking of NMIs.
3879	* See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation".
3880	*/
3881	if (IEM_VMX_IS_PINCTLS_SET(pVCpu, VMX_PIN_CTLS_NMI_EXIT))
3882	fBlockingNmi = false;
3883
3884	/* Clear virtual-NMI blocking, if any, before causing any further exceptions. */
3885	pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false;
3886	}
3887	#endif
3888
3889	/*
3890	* The SVM nested-guest intercept for IRET takes priority over all exceptions,
3891	* The NMI is still held pending (which I assume means blocking of further NMIs
3892	* is in effect).
3893	*
3894	* See AMD spec. 15.9 "Instruction Intercepts".
3895	* See AMD spec. 15.21.9 "NMI Support".
3896	*/
3897	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IRET))
3898	{
3899	Log(("iret: Guest intercept -> #VMEXIT\n"));
3900	IEM_SVM_UPDATE_NRIP(pVCpu);
3901	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IRET, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
3902	}
3903
3904	/*
3905	* Clear NMI blocking, if any, before causing any further exceptions.
3906	* See Intel spec. 6.7.1 "Handling Multiple NMIs".
3907	*/
3908	if (fBlockingNmi)
3909	CPUMClearInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3910
3911	/*
3912	* Call a mode specific worker.
3913	*/
3914	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
3915	return IEM_CIMPL_CALL_1(iemCImpl_iret_real_v8086, enmEffOpSize);
3916	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
3917	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
3918	return IEM_CIMPL_CALL_1(iemCImpl_iret_64bit, enmEffOpSize);
3919	return IEM_CIMPL_CALL_1(iemCImpl_iret_prot, enmEffOpSize);
3920	}
3921
3922
3923	static void iemLoadallSetSelector(PVMCPUCC pVCpu, uint8_t iSegReg, uint16_t uSel)
3924	{
3925	PCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iSegReg);
3926
3927	pHid->Sel = uSel;
3928	pHid->ValidSel = uSel;
3929	pHid->fFlags = CPUMSELREG_FLAGS_VALID;
3930	}
3931
3932
3933	static void iemLoadall286SetDescCache(PVMCPUCC pVCpu, uint8_t iSegReg, uint8_t const *pbMem)
3934	{
3935	PCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iSegReg);
3936
3937	/* The base is in the first three bytes. */
3938	pHid->u64Base = pbMem[0] + (pbMem[1] << 8) + (pbMem[2] << 16);
3939	/* The attributes are in the fourth byte. */
3940	pHid->Attr.u = pbMem[3];
3941	/* The limit is in the last two bytes. */
3942	pHid->u32Limit = pbMem[4] + (pbMem[5] << 8);
3943	}
3944
3945
3946	/**
3947	* Implements 286 LOADALL (286 CPUs only).
3948	*/
3949	IEM_CIMPL_DEF_0(iemCImpl_loadall286)
3950	{
3951	NOREF(cbInstr);
3952
3953	/* Data is loaded from a buffer at 800h. No checks are done on the
3954	* validity of loaded state.
3955	*
3956	* LOADALL only loads the internal CPU state, it does not access any
3957	* GDT, LDT, or similar tables.
3958	*/
3959
3960	if (pVCpu->iem.s.uCpl != 0)
3961	{
3962	Log(("loadall286: CPL must be 0 not %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
3963	return iemRaiseGeneralProtectionFault0(pVCpu);
3964	}
3965
3966	uint8_t const *pbMem = NULL;
3967	uint16_t const *pa16Mem;
3968	uint8_t const *pa8Mem;
3969	RTGCPHYS GCPtrStart = 0x800; /* Fixed table location. */
3970	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pbMem, 0x66, UINT8_MAX, GCPtrStart, IEM_ACCESS_SYS_R, 0);
3971	if (rcStrict != VINF_SUCCESS)
3972	return rcStrict;
3973
3974	/* The MSW is at offset 0x06. */
3975	pa16Mem = (uint16_t const *)(pbMem + 0x06);
3976	/* Even LOADALL can't clear the MSW.PE bit, though it can set it. */
3977	uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0 & ~(X86_CR0_MP \| X86_CR0_EM \| X86_CR0_TS);
3978	uNewCr0 \|= *pa16Mem & (X86_CR0_PE \| X86_CR0_MP \| X86_CR0_EM \| X86_CR0_TS);
3979	uint64_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
3980
3981	CPUMSetGuestCR0(pVCpu, uNewCr0);
3982	Assert(pVCpu->cpum.GstCtx.cr0 == uNewCr0);
3983
3984	/* Inform PGM if mode changed. */
3985	if ((uNewCr0 & X86_CR0_PE) != (uOldCr0 & X86_CR0_PE))
3986	{
3987	int rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
3988	AssertRCReturn(rc, rc);
3989	/* ignore informational status codes */
3990	}
3991	rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
3992	false /* fForce */);
3993
3994	/* TR selector is at offset 0x16. */
3995	pa16Mem = (uint16_t const *)(pbMem + 0x16);
3996	pVCpu->cpum.GstCtx.tr.Sel = pa16Mem[0];
3997	pVCpu->cpum.GstCtx.tr.ValidSel = pa16Mem[0];
3998	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
3999
4000	/* Followed by FLAGS... */
4001	pVCpu->cpum.GstCtx.eflags.u = pa16Mem[1] \| X86_EFL_1;
4002	pVCpu->cpum.GstCtx.ip = pa16Mem[2]; /* ...and IP. */
4003
4004	/* LDT is at offset 0x1C. */
4005	pa16Mem = (uint16_t const *)(pbMem + 0x1C);
4006	pVCpu->cpum.GstCtx.ldtr.Sel = pa16Mem[0];
4007	pVCpu->cpum.GstCtx.ldtr.ValidSel = pa16Mem[0];
4008	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
4009
4010	/* Segment registers are at offset 0x1E. */
4011	pa16Mem = (uint16_t const *)(pbMem + 0x1E);
4012	iemLoadallSetSelector(pVCpu, X86_SREG_DS, pa16Mem[0]);
4013	iemLoadallSetSelector(pVCpu, X86_SREG_SS, pa16Mem[1]);
4014	iemLoadallSetSelector(pVCpu, X86_SREG_CS, pa16Mem[2]);
4015	iemLoadallSetSelector(pVCpu, X86_SREG_ES, pa16Mem[3]);
4016
4017	/* GPRs are at offset 0x26. */
4018	pa16Mem = (uint16_t const *)(pbMem + 0x26);
4019	pVCpu->cpum.GstCtx.di = pa16Mem[0];
4020	pVCpu->cpum.GstCtx.si = pa16Mem[1];
4021	pVCpu->cpum.GstCtx.bp = pa16Mem[2];
4022	pVCpu->cpum.GstCtx.sp = pa16Mem[3];
4023	pVCpu->cpum.GstCtx.bx = pa16Mem[4];
4024	pVCpu->cpum.GstCtx.dx = pa16Mem[5];
4025	pVCpu->cpum.GstCtx.cx = pa16Mem[6];
4026	pVCpu->cpum.GstCtx.ax = pa16Mem[7];
4027
4028	/* Descriptor caches are at offset 0x36, 6 bytes per entry. */
4029	iemLoadall286SetDescCache(pVCpu, X86_SREG_ES, pbMem + 0x36);
4030	iemLoadall286SetDescCache(pVCpu, X86_SREG_CS, pbMem + 0x3C);
4031	iemLoadall286SetDescCache(pVCpu, X86_SREG_SS, pbMem + 0x42);
4032	iemLoadall286SetDescCache(pVCpu, X86_SREG_DS, pbMem + 0x48);
4033
4034	/* GDTR contents are at offset 0x4E, 6 bytes. */
4035	RTGCPHYS GCPtrBase;
4036	uint16_t cbLimit;
4037	pa8Mem = pbMem + 0x4E;
4038	/* NB: Fourth byte "should be zero"; we are ignoring it. */
4039	GCPtrBase = pa8Mem[0] + (pa8Mem[1] << 8) + (pa8Mem[2] << 16);
4040	cbLimit = pa8Mem[4] + (pa8Mem[5] << 8);
4041	CPUMSetGuestGDTR(pVCpu, GCPtrBase, cbLimit);
4042
4043	/* IDTR contents are at offset 0x5A, 6 bytes. */
4044	pa8Mem = pbMem + 0x5A;
4045	GCPtrBase = pa8Mem[0] + (pa8Mem[1] << 8) + (pa8Mem[2] << 16);
4046	cbLimit = pa8Mem[4] + (pa8Mem[5] << 8);
4047	CPUMSetGuestIDTR(pVCpu, GCPtrBase, cbLimit);
4048
4049	Log(("LOADALL: GDTR:%08RX64/%04X, IDTR:%08RX64/%04X\n", pVCpu->cpum.GstCtx.gdtr.pGdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, pVCpu->cpum.GstCtx.idtr.pIdt, pVCpu->cpum.GstCtx.idtr.cbIdt));
4050	Log(("LOADALL: CS:%04X, CS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.cs.u64Base, pVCpu->cpum.GstCtx.cs.u32Limit, pVCpu->cpum.GstCtx.cs.Attr.u));
4051	Log(("LOADALL: DS:%04X, DS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.ds.u64Base, pVCpu->cpum.GstCtx.ds.u32Limit, pVCpu->cpum.GstCtx.ds.Attr.u));
4052	Log(("LOADALL: ES:%04X, ES base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.es.Sel, pVCpu->cpum.GstCtx.es.u64Base, pVCpu->cpum.GstCtx.es.u32Limit, pVCpu->cpum.GstCtx.es.Attr.u));
4053	Log(("LOADALL: SS:%04X, SS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ss.u64Base, pVCpu->cpum.GstCtx.ss.u32Limit, pVCpu->cpum.GstCtx.ss.Attr.u));
4054	Log(("LOADALL: SI:%04X, DI:%04X, AX:%04X, BX:%04X, CX:%04X, DX:%04X\n", pVCpu->cpum.GstCtx.si, pVCpu->cpum.GstCtx.di, pVCpu->cpum.GstCtx.bx, pVCpu->cpum.GstCtx.bx, pVCpu->cpum.GstCtx.cx, pVCpu->cpum.GstCtx.dx));
4055
4056	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pbMem, IEM_ACCESS_SYS_R);
4057	if (rcStrict != VINF_SUCCESS)
4058	return rcStrict;
4059
4060	/* The CPL may change. It is taken from the "DPL fields of the SS and CS
4061	* descriptor caches" but there is no word as to what happens if those are
4062	* not identical (probably bad things).
4063	*/
4064	pVCpu->iem.s.uCpl = pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl;
4065
4066	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_LDTR);
4067
4068	/* Flush the prefetch buffer. */
4069	#ifdef IEM_WITH_CODE_TLB
4070	pVCpu->iem.s.pbInstrBuf = NULL;
4071	#else
4072	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
4073	#endif
4074	return rcStrict;
4075	}
4076
4077
4078	/**
4079	* Implements SYSCALL (AMD and Intel64).
4080	*/
4081	IEM_CIMPL_DEF_0(iemCImpl_syscall)
4082	{
4083	/** @todo hack, LOADALL should be decoded as such on a 286. */
4084	if (RT_UNLIKELY(pVCpu->iem.s.uTargetCpu == IEMTARGETCPU_286))
4085	return iemCImpl_loadall286(pVCpu, cbInstr);
4086
4087	/*
4088	* Check preconditions.
4089	*
4090	* Note that CPUs described in the documentation may load a few odd values
4091	* into CS and SS than we allow here. This has yet to be checked on real
4092	* hardware.
4093	*/
4094	if (!(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SCE))
4095	{
4096	Log(("syscall: Not enabled in EFER -> #UD\n"));
4097	return iemRaiseUndefinedOpcode(pVCpu);
4098	}
4099	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
4100	{
4101	Log(("syscall: Protected mode is required -> #GP(0)\n"));
4102	return iemRaiseGeneralProtectionFault0(pVCpu);
4103	}
4104	if (IEM_IS_GUEST_CPU_INTEL(pVCpu) && !CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4105	{
4106	Log(("syscall: Only available in long mode on intel -> #UD\n"));
4107	return iemRaiseUndefinedOpcode(pVCpu);
4108	}
4109
4110	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS);
4111
4112	/** @todo verify RPL ignoring and CS=0xfff8 (i.e. SS == 0). */
4113	/** @todo what about LDT selectors? Shouldn't matter, really. */
4114	uint16_t uNewCs = (pVCpu->cpum.GstCtx.msrSTAR >> MSR_K6_STAR_SYSCALL_CS_SS_SHIFT) & X86_SEL_MASK_OFF_RPL;
4115	uint16_t uNewSs = uNewCs + 8;
4116	if (uNewCs == 0 \|\| uNewSs == 0)
4117	{
4118	/** @todo Neither Intel nor AMD document this check. */
4119	Log(("syscall: msrSTAR.CS = 0 or SS = 0 -> #GP(0)\n"));
4120	return iemRaiseGeneralProtectionFault0(pVCpu);
4121	}
4122
4123	/* Long mode and legacy mode differs. */
4124	if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4125	{
4126	uint64_t uNewRip = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.msrLSTAR : pVCpu->cpum.GstCtx. msrCSTAR;
4127
4128	/* This test isn't in the docs, but I'm not trusting the guys writing
4129	the MSRs to have validated the values as canonical like they should. */
4130	if (!IEM_IS_CANONICAL(uNewRip))
4131	{
4132	/** @todo Intel claims this can't happen because IA32_LSTAR MSR can't be written with non-canonical address. */
4133	Log(("syscall: New RIP not canonical -> #UD\n"));
4134	return iemRaiseUndefinedOpcode(pVCpu);
4135	}
4136
4137	/*
4138	* Commit it.
4139	*/
4140	Log(("syscall: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, uNewRip));
4141	pVCpu->cpum.GstCtx.rcx = pVCpu->cpum.GstCtx.rip + cbInstr;
4142	pVCpu->cpum.GstCtx.rip = uNewRip;
4143
4144	pVCpu->cpum.GstCtx.rflags.u &= ~X86_EFL_RF;
4145	pVCpu->cpum.GstCtx.r11 = pVCpu->cpum.GstCtx.rflags.u;
4146	pVCpu->cpum.GstCtx.rflags.u &= ~pVCpu->cpum.GstCtx.msrSFMASK;
4147	pVCpu->cpum.GstCtx.rflags.u \|= X86_EFL_1;
4148
4149	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_L \| X86DESCATTR_DT \| X86_SEL_TYPE_ER_ACC;
4150	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_RW_ACC;
4151	}
4152	else
4153	{
4154	/*
4155	* Commit it.
4156	*/
4157	Log(("syscall: %04x:%08RX32 [efl=%#x] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, uNewCs, (uint32_t)(pVCpu->cpum.GstCtx.msrSTAR & MSR_K6_STAR_SYSCALL_EIP_MASK)));
4158	pVCpu->cpum.GstCtx.rcx = pVCpu->cpum.GstCtx.eip + cbInstr;
4159	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.msrSTAR & MSR_K6_STAR_SYSCALL_EIP_MASK;
4160	pVCpu->cpum.GstCtx.rflags.u &= ~(X86_EFL_VM \| X86_EFL_IF \| X86_EFL_RF);
4161
4162	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_ER_ACC;
4163	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_RW_ACC;
4164	}
4165	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
4166	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
4167	pVCpu->cpum.GstCtx.cs.u64Base = 0;
4168	pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX;
4169	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4170
4171	pVCpu->cpum.GstCtx.ss.Sel = uNewSs;
4172	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSs;
4173	pVCpu->cpum.GstCtx.ss.u64Base = 0;
4174	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
4175	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4176
4177	pVCpu->iem.s.uCpl = 0;
4178	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
4179
4180	/* Flush the prefetch buffer. */
4181	#ifdef IEM_WITH_CODE_TLB
4182	pVCpu->iem.s.pbInstrBuf = NULL;
4183	#else
4184	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
4185	#endif
4186
4187	return VINF_SUCCESS;
4188	}
4189
4190
4191	/**
4192	* Implements SYSRET (AMD and Intel64).
4193	*/
4194	IEM_CIMPL_DEF_0(iemCImpl_sysret)
4195
4196	{
4197	RT_NOREF_PV(cbInstr);
4198
4199	/*
4200	* Check preconditions.
4201	*
4202	* Note that CPUs described in the documentation may load a few odd values
4203	* into CS and SS than we allow here. This has yet to be checked on real
4204	* hardware.
4205	*/
4206	if (!(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SCE))
4207	{
4208	Log(("sysret: Not enabled in EFER -> #UD\n"));
4209	return iemRaiseUndefinedOpcode(pVCpu);
4210	}
4211	if (IEM_IS_GUEST_CPU_INTEL(pVCpu) && !CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4212	{
4213	Log(("sysret: Only available in long mode on intel -> #UD\n"));
4214	return iemRaiseUndefinedOpcode(pVCpu);
4215	}
4216	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
4217	{
4218	Log(("sysret: Protected mode is required -> #GP(0)\n"));
4219	return iemRaiseGeneralProtectionFault0(pVCpu);
4220	}
4221	if (pVCpu->iem.s.uCpl != 0)
4222	{
4223	Log(("sysret: CPL must be 0 not %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
4224	return iemRaiseGeneralProtectionFault0(pVCpu);
4225	}
4226
4227	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS);
4228
4229	/** @todo Does SYSRET verify CS != 0 and SS != 0? Neither is valid in ring-3. */
4230	uint16_t uNewCs = (pVCpu->cpum.GstCtx.msrSTAR >> MSR_K6_STAR_SYSRET_CS_SS_SHIFT) & X86_SEL_MASK_OFF_RPL;
4231	uint16_t uNewSs = uNewCs + 8;
4232	if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
4233	uNewCs += 16;
4234	if (uNewCs == 0 \|\| uNewSs == 0)
4235	{
4236	Log(("sysret: msrSTAR.CS = 0 or SS = 0 -> #GP(0)\n"));
4237	return iemRaiseGeneralProtectionFault0(pVCpu);
4238	}
4239
4240	/*
4241	* Commit it.
4242	*/
4243	if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4244	{
4245	if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
4246	{
4247	Log(("sysret: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64 [r11=%#llx]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, pVCpu->cpum.GstCtx.rcx, pVCpu->cpum.GstCtx.r11));
4248	/* Note! We disregard intel manual regarding the RCX canonical
4249	check, ask intel+xen why AMD doesn't do it. */
4250	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rcx;
4251	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_L \| X86DESCATTR_DT \| X86_SEL_TYPE_ER_ACC
4252	\| (3 << X86DESCATTR_DPL_SHIFT);
4253	}
4254	else
4255	{
4256	Log(("sysret: %04x:%016RX64 [efl=%#llx] -> %04x:%08RX32 [r11=%#llx]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.r11));
4257	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.ecx;
4258	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_ER_ACC
4259	\| (3 << X86DESCATTR_DPL_SHIFT);
4260	}
4261	/** @todo testcase: See what kind of flags we can make SYSRET restore and
4262	* what it really ignores. RF and VM are hinted at being zero, by AMD.
4263	* Intel says: RFLAGS := (R11 & 3C7FD7H) \| 2; */
4264	pVCpu->cpum.GstCtx.rflags.u = pVCpu->cpum.GstCtx.r11 & (X86_EFL_POPF_BITS \| X86_EFL_VIF \| X86_EFL_VIP);
4265	pVCpu->cpum.GstCtx.rflags.u \|= X86_EFL_1;
4266	}
4267	else
4268	{
4269	Log(("sysret: %04x:%08RX32 [efl=%#x] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, uNewCs, pVCpu->cpum.GstCtx.ecx));
4270	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rcx;
4271	pVCpu->cpum.GstCtx.rflags.u \|= X86_EFL_IF;
4272	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_ER_ACC
4273	\| (3 << X86DESCATTR_DPL_SHIFT);
4274	}
4275	pVCpu->cpum.GstCtx.cs.Sel = uNewCs \| 3;
4276	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs \| 3;
4277	pVCpu->cpum.GstCtx.cs.u64Base = 0;
4278	pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX;
4279	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4280
4281	pVCpu->cpum.GstCtx.ss.Sel = uNewSs \| 3;
4282	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSs \| 3;
4283	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4284	/* The SS hidden bits remains unchanged says AMD. To that I say "Yeah, right!". */
4285	pVCpu->cpum.GstCtx.ss.Attr.u \|= (3 << X86DESCATTR_DPL_SHIFT);
4286	/** @todo Testcase: verify that SS.u1Long and SS.u1DefBig are left unchanged
4287	* on sysret. */
4288
4289	pVCpu->iem.s.uCpl = 3;
4290	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
4291
4292	/* Flush the prefetch buffer. */
4293	#ifdef IEM_WITH_CODE_TLB
4294	pVCpu->iem.s.pbInstrBuf = NULL;
4295	#else
4296	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
4297	#endif
4298
4299	return VINF_SUCCESS;
4300	}
4301
4302
4303	/**
4304	* Implements SYSENTER (Intel, 32-bit AMD).
4305	*/
4306	IEM_CIMPL_DEF_0(iemCImpl_sysenter)
4307	{
4308	RT_NOREF(cbInstr);
4309
4310	/*
4311	* Check preconditions.
4312	*
4313	* Note that CPUs described in the documentation may load a few odd values
4314	* into CS and SS than we allow here. This has yet to be checked on real
4315	* hardware.
4316	*/
4317	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSysEnter)
4318	{
4319	Log(("sysenter: not supported -=> #UD\n"));
4320	return iemRaiseUndefinedOpcode(pVCpu);
4321	}
4322	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
4323	{
4324	Log(("sysenter: Protected or long mode is required -> #GP(0)\n"));
4325	return iemRaiseGeneralProtectionFault0(pVCpu);
4326	}
4327	bool fIsLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu));
4328	if (IEM_IS_GUEST_CPU_AMD(pVCpu) && fIsLongMode)
4329	{
4330	Log(("sysenter: Only available in protected mode on AMD -> #UD\n"));
4331	return iemRaiseUndefinedOpcode(pVCpu);
4332	}
4333	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4334	uint16_t uNewCs = pVCpu->cpum.GstCtx.SysEnter.cs;
4335	if ((uNewCs & X86_SEL_MASK_OFF_RPL) == 0)
4336	{
4337	Log(("sysenter: SYSENTER_CS = %#x -> #GP(0)\n", uNewCs));
4338	return iemRaiseGeneralProtectionFault0(pVCpu);
4339	}
4340
4341	/* This test isn't in the docs, it's just a safeguard against missing
4342	canonical checks when writing the registers. */
4343	if (RT_LIKELY( !fIsLongMode
4344	\|\| ( IEM_IS_CANONICAL(pVCpu->cpum.GstCtx.SysEnter.eip)
4345	&& IEM_IS_CANONICAL(pVCpu->cpum.GstCtx.SysEnter.esp))))
4346	{ /* likely */ }
4347	else
4348	{
4349	Log(("sysenter: SYSENTER_EIP = %#RX64 or/and SYSENTER_ESP = %#RX64 not canonical -> #GP(0)\n",
4350	pVCpu->cpum.GstCtx.SysEnter.eip, pVCpu->cpum.GstCtx.SysEnter.esp));
4351	return iemRaiseUndefinedOpcode(pVCpu);
4352	}
4353
4354	/** @todo Test: Sysenter from ring-0, ring-1 and ring-2. */
4355
4356	/*
4357	* Update registers and commit.
4358	*/
4359	if (fIsLongMode)
4360	{
4361	Log(("sysenter: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip,
4362	pVCpu->cpum.GstCtx.rflags.u, uNewCs & X86_SEL_MASK_OFF_RPL, pVCpu->cpum.GstCtx.SysEnter.eip));
4363	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.SysEnter.eip;
4364	pVCpu->cpum.GstCtx.rsp = pVCpu->cpum.GstCtx.SysEnter.esp;
4365	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_L \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4366	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_ER_ACC;
4367	}
4368	else
4369	{
4370	Log(("sysenter: %04x:%08RX32 [efl=%#llx] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs, (uint32_t)pVCpu->cpum.GstCtx.rip,
4371	pVCpu->cpum.GstCtx.rflags.u, uNewCs & X86_SEL_MASK_OFF_RPL, (uint32_t)pVCpu->cpum.GstCtx.SysEnter.eip));
4372	pVCpu->cpum.GstCtx.rip = (uint32_t)pVCpu->cpum.GstCtx.SysEnter.eip;
4373	pVCpu->cpum.GstCtx.rsp = (uint32_t)pVCpu->cpum.GstCtx.SysEnter.esp;
4374	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_D \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4375	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_ER_ACC;
4376	}
4377	pVCpu->cpum.GstCtx.cs.Sel = uNewCs & X86_SEL_MASK_OFF_RPL;
4378	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs & X86_SEL_MASK_OFF_RPL;
4379	pVCpu->cpum.GstCtx.cs.u64Base = 0;
4380	pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX;
4381	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4382
4383	pVCpu->cpum.GstCtx.ss.Sel = (uNewCs & X86_SEL_MASK_OFF_RPL) + 8;
4384	pVCpu->cpum.GstCtx.ss.ValidSel = (uNewCs & X86_SEL_MASK_OFF_RPL) + 8;
4385	pVCpu->cpum.GstCtx.ss.u64Base = 0;
4386	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
4387	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_D \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4388	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_RW_ACC;
4389	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4390
4391	pVCpu->cpum.GstCtx.rflags.Bits.u1IF = 0;
4392	pVCpu->cpum.GstCtx.rflags.Bits.u1VM = 0;
4393	pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
4394
4395	pVCpu->iem.s.uCpl = 0;
4396
4397	/* Flush the prefetch buffer. */
4398	#ifdef IEM_WITH_CODE_TLB
4399	pVCpu->iem.s.pbInstrBuf = NULL;
4400	#else
4401	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
4402	#endif
4403
4404	return VINF_SUCCESS;
4405	}
4406
4407
4408	/**
4409	* Implements SYSEXIT (Intel, 32-bit AMD).
4410	*
4411	* @param enmEffOpSize The effective operand size.
4412	*/
4413	IEM_CIMPL_DEF_1(iemCImpl_sysexit, IEMMODE, enmEffOpSize)
4414	{
4415	RT_NOREF(cbInstr);
4416
4417	/*
4418	* Check preconditions.
4419	*
4420	* Note that CPUs described in the documentation may load a few odd values
4421	* into CS and SS than we allow here. This has yet to be checked on real
4422	* hardware.
4423	*/
4424	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSysEnter)
4425	{
4426	Log(("sysexit: not supported -=> #UD\n"));
4427	return iemRaiseUndefinedOpcode(pVCpu);
4428	}
4429	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
4430	{
4431	Log(("sysexit: Protected or long mode is required -> #GP(0)\n"));
4432	return iemRaiseGeneralProtectionFault0(pVCpu);
4433	}
4434	bool fIsLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu));
4435	if (IEM_IS_GUEST_CPU_AMD(pVCpu) && fIsLongMode)
4436	{
4437	Log(("sysexit: Only available in protected mode on AMD -> #UD\n"));
4438	return iemRaiseUndefinedOpcode(pVCpu);
4439	}
4440	if (pVCpu->iem.s.uCpl != 0)
4441	{
4442	Log(("sysexit: CPL(=%u) != 0 -> #GP(0)\n", pVCpu->iem.s.uCpl));
4443	return iemRaiseGeneralProtectionFault0(pVCpu);
4444	}
4445	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4446	uint16_t uNewCs = pVCpu->cpum.GstCtx.SysEnter.cs;
4447	if ((uNewCs & X86_SEL_MASK_OFF_RPL) == 0)
4448	{
4449	Log(("sysexit: SYSENTER_CS = %#x -> #GP(0)\n", uNewCs));
4450	return iemRaiseGeneralProtectionFault0(pVCpu);
4451	}
4452
4453	/*
4454	* Update registers and commit.
4455	*/
4456	if (enmEffOpSize == IEMMODE_64BIT)
4457	{
4458	Log(("sysexit: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip,
4459	pVCpu->cpum.GstCtx.rflags.u, (uNewCs \| 3) + 32, pVCpu->cpum.GstCtx.rcx));
4460	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rdx;
4461	pVCpu->cpum.GstCtx.rsp = pVCpu->cpum.GstCtx.rcx;
4462	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_L \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4463	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_ER_ACC \| (3 << X86DESCATTR_DPL_SHIFT);
4464	pVCpu->cpum.GstCtx.cs.Sel = (uNewCs \| 3) + 32;
4465	pVCpu->cpum.GstCtx.cs.ValidSel = (uNewCs \| 3) + 32;
4466	pVCpu->cpum.GstCtx.ss.Sel = (uNewCs \| 3) + 40;
4467	pVCpu->cpum.GstCtx.ss.ValidSel = (uNewCs \| 3) + 40;
4468	}
4469	else
4470	{
4471	Log(("sysexit: %04x:%08RX64 [efl=%#llx] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip,
4472	pVCpu->cpum.GstCtx.rflags.u, (uNewCs \| 3) + 16, (uint32_t)pVCpu->cpum.GstCtx.edx));
4473	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.edx;
4474	pVCpu->cpum.GstCtx.rsp = pVCpu->cpum.GstCtx.ecx;
4475	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_D \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4476	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_ER_ACC \| (3 << X86DESCATTR_DPL_SHIFT);
4477	pVCpu->cpum.GstCtx.cs.Sel = (uNewCs \| 3) + 16;
4478	pVCpu->cpum.GstCtx.cs.ValidSel = (uNewCs \| 3) + 16;
4479	pVCpu->cpum.GstCtx.ss.Sel = (uNewCs \| 3) + 24;
4480	pVCpu->cpum.GstCtx.ss.ValidSel = (uNewCs \| 3) + 24;
4481	}
4482	pVCpu->cpum.GstCtx.cs.u64Base = 0;
4483	pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX;
4484	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4485
4486	pVCpu->cpum.GstCtx.ss.u64Base = 0;
4487	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
4488	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_D \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4489	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_RW_ACC \| (3 << X86DESCATTR_DPL_SHIFT);
4490	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4491	pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
4492
4493	pVCpu->iem.s.uCpl = 3;
4494
4495	/* Flush the prefetch buffer. */
4496	#ifdef IEM_WITH_CODE_TLB
4497	pVCpu->iem.s.pbInstrBuf = NULL;
4498	#else
4499	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
4500	#endif
4501
4502	return VINF_SUCCESS;
4503	}
4504
4505
4506	/**
4507	* Completes a MOV SReg,XXX or POP SReg instruction.
4508	*
4509	* When not modifying SS or when we're already in an interrupt shadow we
4510	* can update RIP and finish the instruction the normal way.
4511	*
4512	* Otherwise, the MOV/POP SS interrupt shadow that we now enable will block
4513	* both TF and DBx events. The TF will be ignored while the DBx ones will
4514	* be delayed till the next instruction boundrary. For more details see
4515	* @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching Stacks}.
4516	*/
4517	DECLINLINE(VBOXSTRICTRC) iemCImpl_LoadSRegFinish(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iSegReg)
4518	{
4519	if (iSegReg != X86_SREG_SS \|\| CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
4520	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
4521
4522	iemRegAddToRip(pVCpu, cbInstr);
4523	pVCpu->cpum.GstCtx.eflags.uBoth &= ~X86_EFL_RF; /* Shadow int isn't set and DRx is delayed, so only clear RF. */
4524	CPUMSetInInterruptShadowSs(&pVCpu->cpum.GstCtx);
4525
4526	return VINF_SUCCESS;
4527	}
4528
4529
4530	/**
4531	* Common worker for 'pop SReg', 'mov SReg, GReg' and 'lXs GReg, reg/mem'.
4532	*
4533	* @param pVCpu The cross context virtual CPU structure of the calling
4534	* thread.
4535	* @param iSegReg The segment register number (valid).
4536	* @param uSel The new selector value.
4537	*/
4538	static VBOXSTRICTRC iemCImpl_LoadSRegWorker(PVMCPUCC pVCpu, uint8_t iSegReg, uint16_t uSel)
4539	{
4540	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
4541	uint16_t *pSel = iemSRegRef(pVCpu, iSegReg);
4542	PCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iSegReg);
4543
4544	Assert(iSegReg <= X86_SREG_GS && iSegReg != X86_SREG_CS);
4545
4546	/*
4547	* Real mode and V8086 mode are easy.
4548	*/
4549	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
4550	{
4551	*pSel = uSel;
4552	pHid->u64Base = (uint32_t)uSel << 4;
4553	pHid->ValidSel = uSel;
4554	pHid->fFlags = CPUMSELREG_FLAGS_VALID;
4555	#if 0 /* AMD Volume 2, chapter 4.1 - "real mode segmentation" - states that limit and attributes are untouched. */
4556	/** @todo Does the CPU actually load limits and attributes in the
4557	* real/V8086 mode segment load case? It doesn't for CS in far
4558	* jumps... Affects unreal mode. */
4559	pHid->u32Limit = 0xffff;
4560	pHid->Attr.u = 0;
4561	pHid->Attr.n.u1Present = 1;
4562	pHid->Attr.n.u1DescType = 1;
4563	pHid->Attr.n.u4Type = iSegReg != X86_SREG_CS
4564	? X86_SEL_TYPE_RW
4565	: X86_SEL_TYPE_READ \| X86_SEL_TYPE_CODE;
4566	#endif
4567	}
4568	/*
4569	* Protected mode.
4570	*
4571	* Check if it's a null segment selector value first, that's OK for DS, ES,
4572	* FS and GS. If not null, then we have to load and parse the descriptor.
4573	*/
4574	else if (!(uSel & X86_SEL_MASK_OFF_RPL))
4575	{
4576	Assert(iSegReg != X86_SREG_CS); /** @todo testcase for \#UD on MOV CS, ax! */
4577	if (iSegReg == X86_SREG_SS)
4578	{
4579	/* In 64-bit kernel mode, the stack can be 0 because of the way
4580	interrupts are dispatched. AMD seems to have a slighly more
4581	relaxed relationship to SS.RPL than intel does. */
4582	/** @todo We cannot 'mov ss, 3' in 64-bit kernel mode, can we? There is a testcase (bs-cpu-xcpt-1), but double check this! */
4583	if ( pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
4584	\|\| pVCpu->iem.s.uCpl > 2
4585	\|\| ( uSel != pVCpu->iem.s.uCpl
4586	&& !IEM_IS_GUEST_CPU_AMD(pVCpu)) )
4587	{
4588	Log(("load sreg %#x -> invalid stack selector, #GP(0)\n", uSel));
4589	return iemRaiseGeneralProtectionFault0(pVCpu);
4590	}
4591	}
4592
4593	pSel = uSel; / Not RPL, remember :-) */
4594	iemHlpLoadNullDataSelectorProt(pVCpu, pHid, uSel);
4595	if (iSegReg == X86_SREG_SS)
4596	pHid->Attr.u \|= pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT;
4597	}
4598	else
4599	{
4600
4601	/* Fetch the descriptor. */
4602	IEMSELDESC Desc;
4603	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP); /** @todo Correct exception? */
4604	if (rcStrict != VINF_SUCCESS)
4605	return rcStrict;
4606
4607	/* Check GPs first. */
4608	if (!Desc.Legacy.Gen.u1DescType)
4609	{
4610	Log(("load sreg %d (=%#x) - system selector (%#x) -> #GP\n", iSegReg, uSel, Desc.Legacy.Gen.u4Type));
4611	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4612	}
4613	if (iSegReg == X86_SREG_SS) /* SS gets different treatment */
4614	{
4615	if ( (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
4616	\|\| !(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
4617	{
4618	Log(("load sreg SS, %#x - code or read only (%#x) -> #GP\n", uSel, Desc.Legacy.Gen.u4Type));
4619	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4620	}
4621	if ((uSel & X86_SEL_RPL) != pVCpu->iem.s.uCpl)
4622	{
4623	Log(("load sreg SS, %#x - RPL and CPL (%d) differs -> #GP\n", uSel, pVCpu->iem.s.uCpl));
4624	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4625	}
4626	if (Desc.Legacy.Gen.u2Dpl != pVCpu->iem.s.uCpl)
4627	{
4628	Log(("load sreg SS, %#x - DPL (%d) and CPL (%d) differs -> #GP\n", uSel, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
4629	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4630	}
4631	}
4632	else
4633	{
4634	if ((Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
4635	{
4636	Log(("load sreg%u, %#x - execute only segment -> #GP\n", iSegReg, uSel));
4637	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4638	}
4639	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
4640	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
4641	{
4642	#if 0 /* this is what intel says. */
4643	if ( (uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
4644	&& pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)
4645	{
4646	Log(("load sreg%u, %#x - both RPL (%d) and CPL (%d) are greater than DPL (%d) -> #GP\n",
4647	iSegReg, uSel, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl, Desc.Legacy.Gen.u2Dpl));
4648	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4649	}
4650	#else /* this is what makes more sense. */
4651	if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
4652	{
4653	Log(("load sreg%u, %#x - RPL (%d) is greater than DPL (%d) -> #GP\n",
4654	iSegReg, uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl));
4655	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4656	}
4657	if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)
4658	{
4659	Log(("load sreg%u, %#x - CPL (%d) is greater than DPL (%d) -> #GP\n",
4660	iSegReg, uSel, pVCpu->iem.s.uCpl, Desc.Legacy.Gen.u2Dpl));
4661	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4662	}
4663	#endif
4664	}
4665	}
4666
4667	/* Is it there? */
4668	if (!Desc.Legacy.Gen.u1Present)
4669	{
4670	Log(("load sreg%d,%#x - segment not present -> #NP\n", iSegReg, uSel));
4671	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
4672	}
4673
4674	/* The base and limit. */
4675	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
4676	uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
4677
4678	/*
4679	* Ok, everything checked out fine. Now set the accessed bit before
4680	* committing the result into the registers.
4681	*/
4682	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
4683	{
4684	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
4685	if (rcStrict != VINF_SUCCESS)
4686	return rcStrict;
4687	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
4688	}
4689
4690	/* commit */
4691	*pSel = uSel;
4692	pHid->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
4693	pHid->u32Limit = cbLimit;
4694	pHid->u64Base = u64Base;
4695	pHid->ValidSel = uSel;
4696	pHid->fFlags = CPUMSELREG_FLAGS_VALID;
4697
4698	/** @todo check if the hidden bits are loaded correctly for 64-bit
4699	* mode. */
4700	}
4701
4702	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pHid));
4703	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
4704	return VINF_SUCCESS;
4705	}
4706
4707
4708	/**
4709	* Implements 'mov SReg, r/m'.
4710	*
4711	* @param iSegReg The segment register number (valid).
4712	* @param uSel The new selector value.
4713	*/
4714	IEM_CIMPL_DEF_2(iemCImpl_load_SReg, uint8_t, iSegReg, uint16_t, uSel)
4715	{
4716	VBOXSTRICTRC rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
4717	if (rcStrict == VINF_SUCCESS)
4718	rcStrict = iemCImpl_LoadSRegFinish(pVCpu, cbInstr, iSegReg);
4719	return rcStrict;
4720	}
4721
4722
4723	/**
4724	* Implements 'pop SReg'.
4725	*
4726	* @param iSegReg The segment register number (valid).
4727	* @param enmEffOpSize The efficient operand size (valid).
4728	*/
4729	IEM_CIMPL_DEF_2(iemCImpl_pop_Sreg, uint8_t, iSegReg, IEMMODE, enmEffOpSize)
4730	{
4731	VBOXSTRICTRC rcStrict;
4732
4733	/*
4734	* Read the selector off the stack and join paths with mov ss, reg.
4735	*/
4736	RTUINT64U TmpRsp;
4737	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
4738	switch (enmEffOpSize)
4739	{
4740	case IEMMODE_16BIT:
4741	{
4742	uint16_t uSel;
4743	rcStrict = iemMemStackPopU16Ex(pVCpu, &uSel, &TmpRsp);
4744	if (rcStrict == VINF_SUCCESS)
4745	rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
4746	break;
4747	}
4748
4749	case IEMMODE_32BIT:
4750	{
4751	uint32_t u32Value;
4752	rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Value, &TmpRsp);
4753	if (rcStrict == VINF_SUCCESS)
4754	rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, (uint16_t)u32Value);
4755	break;
4756	}
4757
4758	case IEMMODE_64BIT:
4759	{
4760	uint64_t u64Value;
4761	rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Value, &TmpRsp);
4762	if (rcStrict == VINF_SUCCESS)
4763	rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, (uint16_t)u64Value);
4764	break;
4765	}
4766	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4767	}
4768
4769	/*
4770	* If the load succeeded, commit the stack change and finish the instruction.
4771	*/
4772	if (rcStrict == VINF_SUCCESS)
4773	{
4774	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
4775	rcStrict = iemCImpl_LoadSRegFinish(pVCpu, cbInstr, iSegReg);
4776	}
4777
4778	return rcStrict;
4779	}
4780
4781
4782	/**
4783	* Implements lgs, lfs, les, lds & lss.
4784	*/
4785	IEM_CIMPL_DEF_5(iemCImpl_load_SReg_Greg, uint16_t, uSel, uint64_t, offSeg, uint8_t, iSegReg, uint8_t, iGReg, IEMMODE, enmEffOpSize)
4786	{
4787	/*
4788	* Use iemCImpl_LoadSRegWorker to do the tricky segment register loading.
4789	*/
4790	/** @todo verify and test that mov, pop and lXs works the segment
4791	* register loading in the exact same way. */
4792	VBOXSTRICTRC rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
4793	if (rcStrict == VINF_SUCCESS)
4794	{
4795	switch (enmEffOpSize)
4796	{
4797	case IEMMODE_16BIT:
4798	(uint16_t )iemGRegRef(pVCpu, iGReg) = offSeg;
4799	break;
4800	case IEMMODE_32BIT:
4801	case IEMMODE_64BIT:
4802	(uint64_t )iemGRegRef(pVCpu, iGReg) = offSeg;
4803	break;
4804	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4805	}
4806	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
4807	}
4808	return rcStrict;
4809	}
4810
4811
4812	/**
4813	* Helper for VERR, VERW, LAR, and LSL and loads the descriptor into memory.
4814	*
4815	* @retval VINF_SUCCESS on success.
4816	* @retval VINF_IEM_SELECTOR_NOT_OK if the selector isn't ok.
4817	* @retval iemMemFetchSysU64 return value.
4818	*
4819	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4820	* @param uSel The selector value.
4821	* @param fAllowSysDesc Whether system descriptors are OK or not.
4822	* @param pDesc Where to return the descriptor on success.
4823	*/
4824	static VBOXSTRICTRC iemCImpl_LoadDescHelper(PVMCPUCC pVCpu, uint16_t uSel, bool fAllowSysDesc, PIEMSELDESC pDesc)
4825	{
4826	pDesc->Long.au64[0] = 0;
4827	pDesc->Long.au64[1] = 0;
4828
4829	if (!(uSel & X86_SEL_MASK_OFF_RPL)) /** @todo test this on 64-bit. */
4830	return VINF_IEM_SELECTOR_NOT_OK;
4831
4832	/* Within the table limits? */
4833	RTGCPTR GCPtrBase;
4834	if (uSel & X86_SEL_LDT)
4835	{
4836	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
4837	if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
4838	\|\| (uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
4839	return VINF_IEM_SELECTOR_NOT_OK;
4840	GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
4841	}
4842	else
4843	{
4844	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR);
4845	if ((uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
4846	return VINF_IEM_SELECTOR_NOT_OK;
4847	GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
4848	}
4849
4850	/* Fetch the descriptor. */
4851	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
4852	if (rcStrict != VINF_SUCCESS)
4853	return rcStrict;
4854	if (!pDesc->Legacy.Gen.u1DescType)
4855	{
4856	if (!fAllowSysDesc)
4857	return VINF_IEM_SELECTOR_NOT_OK;
4858	if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4859	{
4860	rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 8);
4861	if (rcStrict != VINF_SUCCESS)
4862	return rcStrict;
4863	}
4864
4865	}
4866
4867	return VINF_SUCCESS;
4868	}
4869
4870
4871	/**
4872	* Implements verr (fWrite = false) and verw (fWrite = true).
4873	*/
4874	IEM_CIMPL_DEF_2(iemCImpl_VerX, uint16_t, uSel, bool, fWrite)
4875	{
4876	Assert(!IEM_IS_REAL_OR_V86_MODE(pVCpu));
4877
4878	/** @todo figure whether the accessed bit is set or not. */
4879
4880	bool fAccessible = true;
4881	IEMSELDESC Desc;
4882	VBOXSTRICTRC rcStrict = iemCImpl_LoadDescHelper(pVCpu, uSel, false /fAllowSysDesc/, &Desc);
4883	if (rcStrict == VINF_SUCCESS)
4884	{
4885	/* Check the descriptor, order doesn't matter much here. */
4886	if ( !Desc.Legacy.Gen.u1DescType
4887	\|\| !Desc.Legacy.Gen.u1Present)
4888	fAccessible = false;
4889	else
4890	{
4891	if ( fWrite
4892	? (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE
4893	: (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
4894	fAccessible = false;
4895
4896	/** @todo testcase for the conforming behavior. */
4897	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
4898	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
4899	{
4900	if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
4901	fAccessible = false;
4902	else if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)
4903	fAccessible = false;
4904	}
4905	}
4906
4907	}
4908	else if (rcStrict == VINF_IEM_SELECTOR_NOT_OK)
4909	fAccessible = false;
4910	else
4911	return rcStrict;
4912
4913	/* commit */
4914	pVCpu->cpum.GstCtx.eflags.Bits.u1ZF = fAccessible;
4915
4916	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
4917	}
4918
4919
4920	/**
4921	* Implements LAR and LSL with 64-bit operand size.
4922	*
4923	* @returns VINF_SUCCESS.
4924	* @param pu64Dst Pointer to the destination register.
4925	* @param uSel The selector to load details for.
4926	* @param fIsLar true = LAR, false = LSL.
4927	*/
4928	IEM_CIMPL_DEF_3(iemCImpl_LarLsl_u64, uint64_t *, pu64Dst, uint16_t, uSel, bool, fIsLar)
4929	{
4930	Assert(!IEM_IS_REAL_OR_V86_MODE(pVCpu));
4931
4932	/** @todo figure whether the accessed bit is set or not. */
4933
4934	bool fDescOk = true;
4935	IEMSELDESC Desc;
4936	VBOXSTRICTRC rcStrict = iemCImpl_LoadDescHelper(pVCpu, uSel, true /fAllowSysDesc/, &Desc);
4937	if (rcStrict == VINF_SUCCESS)
4938	{
4939	/*
4940	* Check the descriptor type.
4941	*/
4942	if (!Desc.Legacy.Gen.u1DescType)
4943	{
4944	if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4945	{
4946	if (Desc.Long.Gen.u5Zeros)
4947	fDescOk = false;
4948	else
4949	switch (Desc.Long.Gen.u4Type)
4950	{
4951	/** @todo Intel lists 0 as valid for LSL, verify whether that's correct */
4952	case AMD64_SEL_TYPE_SYS_TSS_AVAIL:
4953	case AMD64_SEL_TYPE_SYS_TSS_BUSY:
4954	case AMD64_SEL_TYPE_SYS_LDT: /** @todo Intel lists this as invalid for LAR, AMD and 32-bit does otherwise. */
4955	break;
4956	case AMD64_SEL_TYPE_SYS_CALL_GATE:
4957	fDescOk = fIsLar;
4958	break;
4959	default:
4960	fDescOk = false;
4961	break;
4962	}
4963	}
4964	else
4965	{
4966	switch (Desc.Long.Gen.u4Type)
4967	{
4968	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
4969	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
4970	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
4971	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
4972	case X86_SEL_TYPE_SYS_LDT:
4973	break;
4974	case X86_SEL_TYPE_SYS_286_CALL_GATE:
4975	case X86_SEL_TYPE_SYS_TASK_GATE:
4976	case X86_SEL_TYPE_SYS_386_CALL_GATE:
4977	fDescOk = fIsLar;
4978	break;
4979	default:
4980	fDescOk = false;
4981	break;
4982	}
4983	}
4984	}
4985	if (fDescOk)
4986	{
4987	/*
4988	* Check the RPL/DPL/CPL interaction..
4989	*/
4990	/** @todo testcase for the conforming behavior. */
4991	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF)) != (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF)
4992	\|\| !Desc.Legacy.Gen.u1DescType)
4993	{
4994	if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
4995	fDescOk = false;
4996	else if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)
4997	fDescOk = false;
4998	}
4999	}
5000
5001	if (fDescOk)
5002	{
5003	/*
5004	* All fine, start committing the result.
5005	*/
5006	if (fIsLar)
5007	*pu64Dst = Desc.Legacy.au32[1] & UINT32_C(0x00ffff00);
5008	else
5009	*pu64Dst = X86DESC_LIMIT_G(&Desc.Legacy);
5010	}
5011
5012	}
5013	else if (rcStrict == VINF_IEM_SELECTOR_NOT_OK)
5014	fDescOk = false;
5015	else
5016	return rcStrict;
5017
5018	/* commit flags value and advance rip. */
5019	pVCpu->cpum.GstCtx.eflags.Bits.u1ZF = fDescOk;
5020	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5021	}
5022
5023
5024	/**
5025	* Implements LAR and LSL with 16-bit operand size.
5026	*
5027	* @returns VINF_SUCCESS.
5028	* @param pu16Dst Pointer to the destination register.
5029	* @param uSel The selector to load details for.
5030	* @param fIsLar true = LAR, false = LSL.
5031	*/
5032	IEM_CIMPL_DEF_3(iemCImpl_LarLsl_u16, uint16_t *, pu16Dst, uint16_t, uSel, bool, fIsLar)
5033	{
5034	uint64_t u64TmpDst = *pu16Dst;
5035	IEM_CIMPL_CALL_3(iemCImpl_LarLsl_u64, &u64TmpDst, uSel, fIsLar);
5036	*pu16Dst = u64TmpDst;
5037	return VINF_SUCCESS;
5038	}
5039
5040
5041	/**
5042	* Implements lgdt.
5043	*
5044	* @param iEffSeg The segment of the new gdtr contents
5045	* @param GCPtrEffSrc The address of the new gdtr contents.
5046	* @param enmEffOpSize The effective operand size.
5047	*/
5048	IEM_CIMPL_DEF_3(iemCImpl_lgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize)
5049	{
5050	if (pVCpu->iem.s.uCpl != 0)
5051	return iemRaiseGeneralProtectionFault0(pVCpu);
5052	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
5053
5054	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5055	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5056	{
5057	Log(("lgdt: Guest intercept -> VM-exit\n"));
5058	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_GDTR_IDTR_ACCESS, VMXINSTRID_LGDT, cbInstr);
5059	}
5060
5061	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_GDTR_WRITES))
5062	{
5063	Log(("lgdt: Guest intercept -> #VMEXIT\n"));
5064	IEM_SVM_UPDATE_NRIP(pVCpu);
5065	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_GDTR_WRITE, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5066	}
5067
5068	/*
5069	* Fetch the limit and base address.
5070	*/
5071	uint16_t cbLimit;
5072	RTGCPTR GCPtrBase;
5073	VBOXSTRICTRC rcStrict = iemMemFetchDataXdtr(pVCpu, &cbLimit, &GCPtrBase, iEffSeg, GCPtrEffSrc, enmEffOpSize);
5074	if (rcStrict == VINF_SUCCESS)
5075	{
5076	if ( pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
5077	\|\| X86_IS_CANONICAL(GCPtrBase))
5078	{
5079	rcStrict = CPUMSetGuestGDTR(pVCpu, GCPtrBase, cbLimit);
5080	if (rcStrict == VINF_SUCCESS)
5081	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5082	}
5083	else
5084	{
5085	Log(("iemCImpl_lgdt: Non-canonical base %04x:%RGv\n", cbLimit, GCPtrBase));
5086	return iemRaiseGeneralProtectionFault0(pVCpu);
5087	}
5088	}
5089	return rcStrict;
5090	}
5091
5092
5093	/**
5094	* Implements sgdt.
5095	*
5096	* @param iEffSeg The segment where to store the gdtr content.
5097	* @param GCPtrEffDst The address where to store the gdtr content.
5098	*/
5099	IEM_CIMPL_DEF_2(iemCImpl_sgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
5100	{
5101	/*
5102	* Join paths with sidt.
5103	* Note! No CPL or V8086 checks here, it's a really sad story, ask Intel if
5104	* you really must know.
5105	*/
5106	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5107	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5108	{
5109	Log(("sgdt: Guest intercept -> VM-exit\n"));
5110	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_GDTR_IDTR_ACCESS, VMXINSTRID_SGDT, cbInstr);
5111	}
5112
5113	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_GDTR_READS))
5114	{
5115	Log(("sgdt: Guest intercept -> #VMEXIT\n"));
5116	IEM_SVM_UPDATE_NRIP(pVCpu);
5117	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_GDTR_READ, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5118	}
5119
5120	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR);
5121	VBOXSTRICTRC rcStrict = iemMemStoreDataXdtr(pVCpu, pVCpu->cpum.GstCtx.gdtr.cbGdt, pVCpu->cpum.GstCtx.gdtr.pGdt, iEffSeg, GCPtrEffDst);
5122	if (rcStrict == VINF_SUCCESS)
5123	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5124	return rcStrict;
5125	}
5126
5127
5128	/**
5129	* Implements lidt.
5130	*
5131	* @param iEffSeg The segment of the new idtr contents
5132	* @param GCPtrEffSrc The address of the new idtr contents.
5133	* @param enmEffOpSize The effective operand size.
5134	*/
5135	IEM_CIMPL_DEF_3(iemCImpl_lidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize)
5136	{
5137	if (pVCpu->iem.s.uCpl != 0)
5138	return iemRaiseGeneralProtectionFault0(pVCpu);
5139	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
5140
5141	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IDTR_WRITES))
5142	{
5143	Log(("lidt: Guest intercept -> #VMEXIT\n"));
5144	IEM_SVM_UPDATE_NRIP(pVCpu);
5145	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IDTR_WRITE, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5146	}
5147
5148	/*
5149	* Fetch the limit and base address.
5150	*/
5151	uint16_t cbLimit;
5152	RTGCPTR GCPtrBase;
5153	VBOXSTRICTRC rcStrict = iemMemFetchDataXdtr(pVCpu, &cbLimit, &GCPtrBase, iEffSeg, GCPtrEffSrc, enmEffOpSize);
5154	if (rcStrict == VINF_SUCCESS)
5155	{
5156	if ( pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
5157	\|\| X86_IS_CANONICAL(GCPtrBase))
5158	{
5159	CPUMSetGuestIDTR(pVCpu, GCPtrBase, cbLimit);
5160	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5161	}
5162	else
5163	{
5164	Log(("iemCImpl_lidt: Non-canonical base %04x:%RGv\n", cbLimit, GCPtrBase));
5165	return iemRaiseGeneralProtectionFault0(pVCpu);
5166	}
5167	}
5168	return rcStrict;
5169	}
5170
5171
5172	/**
5173	* Implements sidt.
5174	*
5175	* @param iEffSeg The segment where to store the idtr content.
5176	* @param GCPtrEffDst The address where to store the idtr content.
5177	*/
5178	IEM_CIMPL_DEF_2(iemCImpl_sidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
5179	{
5180	/*
5181	* Join paths with sgdt.
5182	* Note! No CPL or V8086 checks here, it's a really sad story, ask Intel if
5183	* you really must know.
5184	*/
5185	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IDTR_READS))
5186	{
5187	Log(("sidt: Guest intercept -> #VMEXIT\n"));
5188	IEM_SVM_UPDATE_NRIP(pVCpu);
5189	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IDTR_READ, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5190	}
5191
5192	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_IDTR);
5193	VBOXSTRICTRC rcStrict = iemMemStoreDataXdtr(pVCpu, pVCpu->cpum.GstCtx.idtr.cbIdt, pVCpu->cpum.GstCtx.idtr.pIdt, iEffSeg, GCPtrEffDst);
5194	if (rcStrict == VINF_SUCCESS)
5195	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5196	return rcStrict;
5197	}
5198
5199
5200	/**
5201	* Implements lldt.
5202	*
5203	* @param uNewLdt The new LDT selector value.
5204	*/
5205	IEM_CIMPL_DEF_1(iemCImpl_lldt, uint16_t, uNewLdt)
5206	{
5207	/*
5208	* Check preconditions.
5209	*/
5210	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
5211	{
5212	Log(("lldt %04x - real or v8086 mode -> #GP(0)\n", uNewLdt));
5213	return iemRaiseUndefinedOpcode(pVCpu);
5214	}
5215	if (pVCpu->iem.s.uCpl != 0)
5216	{
5217	Log(("lldt %04x - CPL is %d -> #GP(0)\n", uNewLdt, pVCpu->iem.s.uCpl));
5218	return iemRaiseGeneralProtectionFault0(pVCpu);
5219	}
5220	/* Nested-guest VMX intercept. */
5221	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5222	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5223	{
5224	Log(("lldt: Guest intercept -> VM-exit\n"));
5225	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_LLDT, cbInstr);
5226	}
5227	if (uNewLdt & X86_SEL_LDT)
5228	{
5229	Log(("lldt %04x - LDT selector -> #GP\n", uNewLdt));
5230	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewLdt);
5231	}
5232
5233	/*
5234	* Now, loading a NULL selector is easy.
5235	*/
5236	if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
5237	{
5238	/* Nested-guest SVM intercept. */
5239	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_LDTR_WRITES))
5240	{
5241	Log(("lldt: Guest intercept -> #VMEXIT\n"));
5242	IEM_SVM_UPDATE_NRIP(pVCpu);
5243	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_LDTR_WRITE, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5244	}
5245
5246	Log(("lldt %04x: Loading NULL selector.\n", uNewLdt));
5247	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_LDTR;
5248	CPUMSetGuestLDTR(pVCpu, uNewLdt);
5249	pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
5250	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
5251	if (IEM_IS_GUEST_CPU_AMD(pVCpu))
5252	{
5253	/* AMD-V seems to leave the base and limit alone. */
5254	pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESCATTR_UNUSABLE;
5255	}
5256	else
5257	{
5258	/* VT-x (Intel 3960x) seems to be doing the following. */
5259	pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D;
5260	pVCpu->cpum.GstCtx.ldtr.u64Base = 0;
5261	pVCpu->cpum.GstCtx.ldtr.u32Limit = UINT32_MAX;
5262	}
5263
5264	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5265	}
5266
5267	/*
5268	* Read the descriptor.
5269	*/
5270	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_GDTR);
5271	IEMSELDESC Desc;
5272	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uNewLdt, X86_XCPT_GP); /** @todo Correct exception? */
5273	if (rcStrict != VINF_SUCCESS)
5274	return rcStrict;
5275
5276	/* Check GPs first. */
5277	if (Desc.Legacy.Gen.u1DescType)
5278	{
5279	Log(("lldt %#x - not system selector (type %x) -> #GP\n", uNewLdt, Desc.Legacy.Gen.u4Type));
5280	return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
5281	}
5282	if (Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
5283	{
5284	Log(("lldt %#x - not LDT selector (type %x) -> #GP\n", uNewLdt, Desc.Legacy.Gen.u4Type));
5285	return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
5286	}
5287	uint64_t u64Base;
5288	if (!IEM_IS_LONG_MODE(pVCpu))
5289	u64Base = X86DESC_BASE(&Desc.Legacy);
5290	else
5291	{
5292	if (Desc.Long.Gen.u5Zeros)
5293	{
5294	Log(("lldt %#x - u5Zeros=%#x -> #GP\n", uNewLdt, Desc.Long.Gen.u5Zeros));
5295	return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
5296	}
5297
5298	u64Base = X86DESC64_BASE(&Desc.Long);
5299	if (!IEM_IS_CANONICAL(u64Base))
5300	{
5301	Log(("lldt %#x - non-canonical base address %#llx -> #GP\n", uNewLdt, u64Base));
5302	return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
5303	}
5304	}
5305
5306	/* NP */
5307	if (!Desc.Legacy.Gen.u1Present)
5308	{
5309	Log(("lldt %#x - segment not present -> #NP\n", uNewLdt));
5310	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewLdt);
5311	}
5312
5313	/* Nested-guest SVM intercept. */
5314	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_LDTR_WRITES))
5315	{
5316	Log(("lldt: Guest intercept -> #VMEXIT\n"));
5317	IEM_SVM_UPDATE_NRIP(pVCpu);
5318	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_LDTR_WRITE, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5319	}
5320
5321	/*
5322	* It checks out alright, update the registers.
5323	*/
5324	/** @todo check if the actual value is loaded or if the RPL is dropped */
5325	CPUMSetGuestLDTR(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
5326	pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt & X86_SEL_MASK_OFF_RPL;
5327	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
5328	pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
5329	pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&Desc.Legacy);
5330	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
5331
5332	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5333	}
5334
5335
5336	/**
5337	* Implements sldt GReg
5338	*
5339	* @param iGReg The general register to store the CRx value in.
5340	* @param enmEffOpSize The operand size.
5341	*/
5342	IEM_CIMPL_DEF_2(iemCImpl_sldt_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
5343	{
5344	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5345	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5346	{
5347	Log(("sldt: Guest intercept -> VM-exit\n"));
5348	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_SLDT, cbInstr);
5349	}
5350
5351	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_LDTR_READS, SVM_EXIT_LDTR_READ, 0, 0);
5352
5353	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
5354	switch (enmEffOpSize)
5355	{
5356	case IEMMODE_16BIT: (uint16_t )iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break;
5357	case IEMMODE_32BIT: (uint64_t )iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break;
5358	case IEMMODE_64BIT: (uint64_t )iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break;
5359	IEM_NOT_REACHED_DEFAULT_CASE_RET();
5360	}
5361	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5362	}
5363
5364
5365	/**
5366	* Implements sldt mem.
5367	*
5368	* @param iEffSeg The effective segment register to use with @a GCPtrMem.
5369	* @param GCPtrEffDst Where to store the 16-bit CR0 value.
5370	*/
5371	IEM_CIMPL_DEF_2(iemCImpl_sldt_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
5372	{
5373	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_LDTR_READS, SVM_EXIT_LDTR_READ, 0, 0);
5374
5375	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
5376	VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, pVCpu->cpum.GstCtx.ldtr.Sel);
5377	if (rcStrict == VINF_SUCCESS)
5378	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5379	return rcStrict;
5380	}
5381
5382
5383	/**
5384	* Implements ltr.
5385	*
5386	* @param uNewTr The new TSS selector value.
5387	*/
5388	IEM_CIMPL_DEF_1(iemCImpl_ltr, uint16_t, uNewTr)
5389	{
5390	/*
5391	* Check preconditions.
5392	*/
5393	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
5394	{
5395	Log(("ltr %04x - real or v8086 mode -> #GP(0)\n", uNewTr));
5396	return iemRaiseUndefinedOpcode(pVCpu);
5397	}
5398	if (pVCpu->iem.s.uCpl != 0)
5399	{
5400	Log(("ltr %04x - CPL is %d -> #GP(0)\n", uNewTr, pVCpu->iem.s.uCpl));
5401	return iemRaiseGeneralProtectionFault0(pVCpu);
5402	}
5403	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5404	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5405	{
5406	Log(("ltr: Guest intercept -> VM-exit\n"));
5407	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_LTR, cbInstr);
5408	}
5409	if (uNewTr & X86_SEL_LDT)
5410	{
5411	Log(("ltr %04x - LDT selector -> #GP\n", uNewTr));
5412	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewTr);
5413	}
5414	if (!(uNewTr & X86_SEL_MASK_OFF_RPL))
5415	{
5416	Log(("ltr %04x - NULL selector -> #GP(0)\n", uNewTr));
5417	return iemRaiseGeneralProtectionFault0(pVCpu);
5418	}
5419	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TR_WRITES))
5420	{
5421	Log(("ltr: Guest intercept -> #VMEXIT\n"));
5422	IEM_SVM_UPDATE_NRIP(pVCpu);
5423	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TR_WRITE, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5424	}
5425
5426	/*
5427	* Read the descriptor.
5428	*/
5429	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_TR);
5430	IEMSELDESC Desc;
5431	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uNewTr, X86_XCPT_GP); /** @todo Correct exception? */
5432	if (rcStrict != VINF_SUCCESS)
5433	return rcStrict;
5434
5435	/* Check GPs first. */
5436	if (Desc.Legacy.Gen.u1DescType)
5437	{
5438	Log(("ltr %#x - not system selector (type %x) -> #GP\n", uNewTr, Desc.Legacy.Gen.u4Type));
5439	return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
5440	}
5441	if ( Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL /* same as AMD64_SEL_TYPE_SYS_TSS_AVAIL */
5442	&& ( Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
5443	\|\| IEM_IS_LONG_MODE(pVCpu)) )
5444	{
5445	Log(("ltr %#x - not an available TSS selector (type %x) -> #GP\n", uNewTr, Desc.Legacy.Gen.u4Type));
5446	return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
5447	}
5448	uint64_t u64Base;
5449	if (!IEM_IS_LONG_MODE(pVCpu))
5450	u64Base = X86DESC_BASE(&Desc.Legacy);
5451	else
5452	{
5453	if (Desc.Long.Gen.u5Zeros)
5454	{
5455	Log(("ltr %#x - u5Zeros=%#x -> #GP\n", uNewTr, Desc.Long.Gen.u5Zeros));
5456	return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
5457	}
5458
5459	u64Base = X86DESC64_BASE(&Desc.Long);
5460	if (!IEM_IS_CANONICAL(u64Base))
5461	{
5462	Log(("ltr %#x - non-canonical base address %#llx -> #GP\n", uNewTr, u64Base));
5463	return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
5464	}
5465	}
5466
5467	/* NP */
5468	if (!Desc.Legacy.Gen.u1Present)
5469	{
5470	Log(("ltr %#x - segment not present -> #NP\n", uNewTr));
5471	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewTr);
5472	}
5473
5474	/*
5475	* Set it busy.
5476	* Note! Intel says this should lock down the whole descriptor, but we'll
5477	* restrict our selves to 32-bit for now due to lack of inline
5478	* assembly and such.
5479	*/
5480	void *pvDesc;
5481	rcStrict = iemMemMap(pVCpu, &pvDesc, 8, UINT8_MAX, pVCpu->cpum.GstCtx.gdtr.pGdt + (uNewTr & X86_SEL_MASK_OFF_RPL),
5482	IEM_ACCESS_DATA_RW, 0);
5483	if (rcStrict != VINF_SUCCESS)
5484	return rcStrict;
5485	switch ((uintptr_t)pvDesc & 3)
5486	{
5487	case 0: ASMAtomicBitSet(pvDesc, 40 + 1); break;
5488	case 1: ASMAtomicBitSet((uint8_t *)pvDesc + 3, 40 + 1 - 24); break;
5489	case 2: ASMAtomicBitSet((uint8_t *)pvDesc + 2, 40 + 1 - 16); break;
5490	case 3: ASMAtomicBitSet((uint8_t *)pvDesc + 1, 40 + 1 - 8); break;
5491	}
5492	rcStrict = iemMemCommitAndUnmap(pVCpu, pvDesc, IEM_ACCESS_DATA_RW);
5493	if (rcStrict != VINF_SUCCESS)
5494	return rcStrict;
5495	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
5496
5497	/*
5498	* It checks out alright, update the registers.
5499	*/
5500	/** @todo check if the actual value is loaded or if the RPL is dropped */
5501	CPUMSetGuestTR(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
5502	pVCpu->cpum.GstCtx.tr.ValidSel = uNewTr & X86_SEL_MASK_OFF_RPL;
5503	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
5504	pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
5505	pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&Desc.Legacy);
5506	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
5507
5508	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5509	}
5510
5511
5512	/**
5513	* Implements str GReg
5514	*
5515	* @param iGReg The general register to store the CRx value in.
5516	* @param enmEffOpSize The operand size.
5517	*/
5518	IEM_CIMPL_DEF_2(iemCImpl_str_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
5519	{
5520	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5521	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5522	{
5523	Log(("str_reg: Guest intercept -> VM-exit\n"));
5524	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_STR, cbInstr);
5525	}
5526
5527	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_TR_READS, SVM_EXIT_TR_READ, 0, 0);
5528
5529	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
5530	switch (enmEffOpSize)
5531	{
5532	case IEMMODE_16BIT: (uint16_t )iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break;
5533	case IEMMODE_32BIT: (uint64_t )iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break;
5534	case IEMMODE_64BIT: (uint64_t )iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break;
5535	IEM_NOT_REACHED_DEFAULT_CASE_RET();
5536	}
5537	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5538	}
5539
5540
5541	/**
5542	* Implements str mem.
5543	*
5544	* @param iEffSeg The effective segment register to use with @a GCPtrMem.
5545	* @param GCPtrEffDst Where to store the 16-bit CR0 value.
5546	*/
5547	IEM_CIMPL_DEF_2(iemCImpl_str_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
5548	{
5549	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5550	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5551	{
5552	Log(("str_mem: Guest intercept -> VM-exit\n"));
5553	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_STR, cbInstr);
5554	}
5555
5556	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_TR_READS, SVM_EXIT_TR_READ, 0, 0);
5557
5558	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
5559	VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, pVCpu->cpum.GstCtx.tr.Sel);
5560	if (rcStrict == VINF_SUCCESS)
5561	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5562	return rcStrict;
5563	}
5564
5565
5566	/**
5567	* Implements mov GReg,CRx.
5568	*
5569	* @param iGReg The general register to store the CRx value in.
5570	* @param iCrReg The CRx register to read (valid).
5571	*/
5572	IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Cd, uint8_t, iGReg, uint8_t, iCrReg)
5573	{
5574	if (pVCpu->iem.s.uCpl != 0)
5575	return iemRaiseGeneralProtectionFault0(pVCpu);
5576	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
5577
5578	if (IEM_SVM_IS_READ_CR_INTERCEPT_SET(pVCpu, iCrReg))
5579	{
5580	Log(("iemCImpl_mov_Rd_Cd: Guest intercept CR%u -> #VMEXIT\n", iCrReg));
5581	IEM_SVM_UPDATE_NRIP(pVCpu);
5582	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_READ_CR0 + iCrReg, IEMACCESSCRX_MOV_CRX, iGReg);
5583	}
5584
5585	/* Read it. */
5586	uint64_t crX;
5587	switch (iCrReg)
5588	{
5589	case 0:
5590	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5591	crX = pVCpu->cpum.GstCtx.cr0;
5592	if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
5593	crX \|= UINT32_C(0x7fffffe0); /* All reserved CR0 flags are set on a 386, just like MSW on 286. */
5594	break;
5595	case 2:
5596	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR2);
5597	crX = pVCpu->cpum.GstCtx.cr2;
5598	break;
5599	case 3:
5600	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5601	crX = pVCpu->cpum.GstCtx.cr3;
5602	break;
5603	case 4:
5604	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5605	crX = pVCpu->cpum.GstCtx.cr4;
5606	break;
5607	case 8:
5608	{
5609	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5610	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5611	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5612	{
5613	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovFromCr8(pVCpu, iGReg, cbInstr);
5614	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
5615	return rcStrict;
5616
5617	/*
5618	* If the Mov-from-CR8 doesn't cause a VM-exit, bits 7:4 of the VTPR is copied
5619	* to bits 0:3 of the destination operand. Bits 63:4 of the destination operand
5620	* are cleared.
5621	*
5622	* See Intel Spec. 29.3 "Virtualizing CR8-based TPR Accesses"
5623	*/
5624	if (IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_USE_TPR_SHADOW))
5625	{
5626	uint32_t const uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
5627	crX = (uTpr >> 4) & 0xf;
5628	break;
5629	}
5630	}
5631	#endif
5632	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5633	if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
5634	{
5635	PCSVMVMCBCTRL pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl;
5636	if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, IEM_GET_CTX(pVCpu)))
5637	{
5638	crX = pVmcbCtrl->IntCtrl.n.u8VTPR & 0xf;
5639	break;
5640	}
5641	}
5642	#endif
5643	uint8_t uTpr;
5644	int rc = APICGetTpr(pVCpu, &uTpr, NULL, NULL);
5645	if (RT_SUCCESS(rc))
5646	crX = uTpr >> 4;
5647	else
5648	crX = 0;
5649	break;
5650	}
5651	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
5652	}
5653
5654	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5655	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5656	{
5657	switch (iCrReg)
5658	{
5659	/* CR0/CR4 reads are subject to masking when in VMX non-root mode. */
5660	case 0: crX = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u); break;
5661	case 4: crX = CPUMGetGuestVmxMaskedCr4(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr4Mask.u); break;
5662
5663	case 3:
5664	{
5665	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovFromCr3(pVCpu, iGReg, cbInstr);
5666	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
5667	return rcStrict;
5668	break;
5669	}
5670	}
5671	}
5672	#endif
5673
5674	/* Store it. */
5675	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
5676	(uint64_t )iemGRegRef(pVCpu, iGReg) = crX;
5677	else
5678	(uint64_t )iemGRegRef(pVCpu, iGReg) = (uint32_t)crX;
5679
5680	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5681	}
5682
5683
5684	/**
5685	* Implements smsw GReg.
5686	*
5687	* @param iGReg The general register to store the CRx value in.
5688	* @param enmEffOpSize The operand size.
5689	*/
5690	IEM_CIMPL_DEF_2(iemCImpl_smsw_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
5691	{
5692	IEM_SVM_CHECK_READ_CR0_INTERCEPT(pVCpu, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5693
5694	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5695	uint64_t u64MaskedCr0;
5696	if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5697	u64MaskedCr0 = pVCpu->cpum.GstCtx.cr0;
5698	else
5699	u64MaskedCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u);
5700	uint64_t const u64GuestCr0 = u64MaskedCr0;
5701	#else
5702	uint64_t const u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5703	#endif
5704
5705	switch (enmEffOpSize)
5706	{
5707	case IEMMODE_16BIT:
5708	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
5709	(uint16_t )iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0;
5710	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
5711	(uint16_t )iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0 \| 0xffe0;
5712	else
5713	(uint16_t )iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0 \| 0xfff0;
5714	break;
5715
5716	case IEMMODE_32BIT:
5717	(uint32_t )iemGRegRef(pVCpu, iGReg) = (uint32_t)u64GuestCr0;
5718	break;
5719
5720	case IEMMODE_64BIT:
5721	(uint64_t )iemGRegRef(pVCpu, iGReg) = u64GuestCr0;
5722	break;
5723
5724	IEM_NOT_REACHED_DEFAULT_CASE_RET();
5725	}
5726
5727	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5728	}
5729
5730
5731	/**
5732	* Implements smsw mem.
5733	*
5734	* @param iEffSeg The effective segment register to use with @a GCPtrMem.
5735	* @param GCPtrEffDst Where to store the 16-bit CR0 value.
5736	*/
5737	IEM_CIMPL_DEF_2(iemCImpl_smsw_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
5738	{
5739	IEM_SVM_CHECK_READ_CR0_INTERCEPT(pVCpu, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5740
5741	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5742	uint64_t u64MaskedCr0;
5743	if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5744	u64MaskedCr0 = pVCpu->cpum.GstCtx.cr0;
5745	else
5746	u64MaskedCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u);
5747	uint64_t const u64GuestCr0 = u64MaskedCr0;
5748	#else
5749	uint64_t const u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5750	#endif
5751
5752	uint16_t u16Value;
5753	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
5754	u16Value = (uint16_t)u64GuestCr0;
5755	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
5756	u16Value = (uint16_t)u64GuestCr0 \| 0xffe0;
5757	else
5758	u16Value = (uint16_t)u64GuestCr0 \| 0xfff0;
5759
5760	VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, u16Value);
5761	if (rcStrict == VINF_SUCCESS)
5762	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5763	return rcStrict;
5764	}
5765
5766
5767	/**
5768	* Helper for mapping CR3 and PAE PDPEs for 'mov CRx,GReg'.
5769	*/
5770	#define IEM_MAP_PAE_PDPES_AT_CR3_RET(a_pVCpu, a_iCrReg, a_uCr3) \
5771	do \
5772	{ \
5773	int const rcX = PGMGstMapPaePdpesAtCr3(a_pVCpu, a_uCr3); \
5774	if (RT_SUCCESS(rcX)) \
5775	{ /* likely */ } \
5776	else \
5777	{ \
5778	/* Either invalid PDPTEs or CR3 second-level translation failed. Raise #GP(0) either way. */ \
5779	Log(("iemCImpl_load_Cr%#x: Trying to load invalid PAE PDPEs\n", a_iCrReg)); \
5780	return iemRaiseGeneralProtectionFault0(a_pVCpu); \
5781	} \
5782	} while (0)
5783
5784
5785	/**
5786	* Used to implemented 'mov CRx,GReg' and 'lmsw r/m16'.
5787	*
5788	* @param iCrReg The CRx register to write (valid).
5789	* @param uNewCrX The new value.
5790	* @param enmAccessCrX The instruction that caused the CrX load.
5791	* @param iGReg The general register in case of a 'mov CRx,GReg'
5792	* instruction.
5793	*/
5794	IEM_CIMPL_DEF_4(iemCImpl_load_CrX, uint8_t, iCrReg, uint64_t, uNewCrX, IEMACCESSCRX, enmAccessCrX, uint8_t, iGReg)
5795	{
5796	VBOXSTRICTRC rcStrict;
5797	int rc;
5798	#ifndef VBOX_WITH_NESTED_HWVIRT_SVM
5799	RT_NOREF2(iGReg, enmAccessCrX);
5800	#endif
5801
5802	/*
5803	* Try store it.
5804	* Unfortunately, CPUM only does a tiny bit of the work.
5805	*/
5806	switch (iCrReg)
5807	{
5808	case 0:
5809	{
5810	/*
5811	* Perform checks.
5812	*/
5813	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5814
5815	uint64_t const uOldCrX = pVCpu->cpum.GstCtx.cr0;
5816	uint32_t const fValid = CPUMGetGuestCR0ValidMask();
5817
5818	/* ET is hardcoded on 486 and later. */
5819	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_486)
5820	uNewCrX \|= X86_CR0_ET;
5821	/* The 386 and 486 didn't #GP(0) on attempting to set reserved CR0 bits. ET was settable on 386. */
5822	else if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_486)
5823	{
5824	uNewCrX &= fValid;
5825	uNewCrX \|= X86_CR0_ET;
5826	}
5827	else
5828	uNewCrX &= X86_CR0_PE \| X86_CR0_MP \| X86_CR0_EM \| X86_CR0_TS \| X86_CR0_PG \| X86_CR0_ET;
5829
5830	/* Check for reserved bits. */
5831	if (uNewCrX & ~(uint64_t)fValid)
5832	{
5833	Log(("Trying to set reserved CR0 bits: NewCR0=%#llx InvalidBits=%#llx\n", uNewCrX, uNewCrX & ~(uint64_t)fValid));
5834	return iemRaiseGeneralProtectionFault0(pVCpu);
5835	}
5836
5837	/* Check for invalid combinations. */
5838	if ( (uNewCrX & X86_CR0_PG)
5839	&& !(uNewCrX & X86_CR0_PE) )
5840	{
5841	Log(("Trying to set CR0.PG without CR0.PE\n"));
5842	return iemRaiseGeneralProtectionFault0(pVCpu);
5843	}
5844
5845	if ( !(uNewCrX & X86_CR0_CD)
5846	&& (uNewCrX & X86_CR0_NW) )
5847	{
5848	Log(("Trying to clear CR0.CD while leaving CR0.NW set\n"));
5849	return iemRaiseGeneralProtectionFault0(pVCpu);
5850	}
5851
5852	if ( !(uNewCrX & X86_CR0_PG)
5853	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCIDE))
5854	{
5855	Log(("Trying to clear CR0.PG while leaving CR4.PCID set\n"));
5856	return iemRaiseGeneralProtectionFault0(pVCpu);
5857	}
5858
5859	/* Long mode consistency checks. */
5860	if ( (uNewCrX & X86_CR0_PG)
5861	&& !(uOldCrX & X86_CR0_PG)
5862	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME) )
5863	{
5864	if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE))
5865	{
5866	Log(("Trying to enabled long mode paging without CR4.PAE set\n"));
5867	return iemRaiseGeneralProtectionFault0(pVCpu);
5868	}
5869	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1Long)
5870	{
5871	Log(("Trying to enabled long mode paging with a long CS descriptor loaded.\n"));
5872	return iemRaiseGeneralProtectionFault0(pVCpu);
5873	}
5874	}
5875
5876	/* Check for bits that must remain set or cleared in VMX operation,
5877	see Intel spec. 23.8 "Restrictions on VMX operation". */
5878	if (IEM_VMX_IS_ROOT_MODE(pVCpu))
5879	{
5880	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5881	uint64_t const uCr0Fixed0 = IEM_VMX_IS_NON_ROOT_MODE(pVCpu) ? iemVmxGetCr0Fixed0(pVCpu) : VMX_V_CR0_FIXED0;
5882	#else
5883	uint64_t const uCr0Fixed0 = VMX_V_CR0_FIXED0;
5884	#endif
5885	if ((uNewCrX & uCr0Fixed0) != uCr0Fixed0)
5886	{
5887	Log(("Trying to clear reserved CR0 bits in VMX operation: NewCr0=%#llx MB1=%#llx\n", uNewCrX, uCr0Fixed0));
5888	return iemRaiseGeneralProtectionFault0(pVCpu);
5889	}
5890
5891	uint64_t const uCr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
5892	if (uNewCrX & ~uCr0Fixed1)
5893	{
5894	Log(("Trying to set reserved CR0 bits in VMX operation: NewCr0=%#llx MB0=%#llx\n", uNewCrX, uCr0Fixed1));
5895	return iemRaiseGeneralProtectionFault0(pVCpu);
5896	}
5897	}
5898
5899	/*
5900	* SVM nested-guest CR0 write intercepts.
5901	*/
5902	if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, iCrReg))
5903	{
5904	Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
5905	IEM_SVM_UPDATE_NRIP(pVCpu);
5906	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR0, enmAccessCrX, iGReg);
5907	}
5908	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5909	{
5910	/* 'lmsw' intercepts regardless of whether the TS/MP bits are actually toggled. */
5911	if ( enmAccessCrX == IEMACCESSCRX_LMSW
5912	\|\| (uNewCrX & ~(X86_CR0_TS \| X86_CR0_MP)) != (uOldCrX & ~(X86_CR0_TS \| X86_CR0_MP)))
5913	{
5914	Assert(enmAccessCrX != IEMACCESSCRX_CLTS);
5915	Log(("iemCImpl_load_Cr%#x: lmsw or bits other than TS/MP changed: Guest intercept -> #VMEXIT\n", iCrReg));
5916	IEM_SVM_UPDATE_NRIP(pVCpu);
5917	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_CR0_SEL_WRITE, enmAccessCrX, iGReg);
5918	}
5919	}
5920
5921	/*
5922	* Change EFER.LMA if entering or leaving long mode.
5923	*/
5924	uint64_t NewEFER = pVCpu->cpum.GstCtx.msrEFER;
5925	if ( (uNewCrX & X86_CR0_PG) != (uOldCrX & X86_CR0_PG)
5926	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME) )
5927	{
5928	if (uNewCrX & X86_CR0_PG)
5929	NewEFER \|= MSR_K6_EFER_LMA;
5930	else
5931	NewEFER &= ~MSR_K6_EFER_LMA;
5932
5933	CPUMSetGuestEFER(pVCpu, NewEFER);
5934	Assert(pVCpu->cpum.GstCtx.msrEFER == NewEFER);
5935	}
5936
5937	/*
5938	* Inform PGM.
5939	*/
5940	if ( (uNewCrX & (X86_CR0_PG \| X86_CR0_WP \| X86_CR0_PE \| X86_CR0_CD \| X86_CR0_NW))
5941	!= (uOldCrX & (X86_CR0_PG \| X86_CR0_WP \| X86_CR0_PE \| X86_CR0_CD \| X86_CR0_NW)) )
5942	{
5943	if ( enmAccessCrX != IEMACCESSCRX_MOV_CRX
5944	\|\| !CPUMIsPaePagingEnabled(uNewCrX, pVCpu->cpum.GstCtx.cr4, NewEFER)
5945	\|\| CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
5946	{ /* likely */ }
5947	else
5948	IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, pVCpu->cpum.GstCtx.cr3);
5949	rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
5950	AssertRCReturn(rc, rc);
5951	/* ignore informational status codes */
5952	}
5953
5954	/*
5955	* Change CR0.
5956	*/
5957	CPUMSetGuestCR0(pVCpu, uNewCrX);
5958	Assert(pVCpu->cpum.GstCtx.cr0 == uNewCrX);
5959
5960	rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
5961	false /* fForce */);
5962	break;
5963	}
5964
5965	/*
5966	* CR2 can be changed without any restrictions.
5967	*/
5968	case 2:
5969	{
5970	if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /cr/ 2))
5971	{
5972	Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
5973	IEM_SVM_UPDATE_NRIP(pVCpu);
5974	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR2, enmAccessCrX, iGReg);
5975	}
5976	pVCpu->cpum.GstCtx.cr2 = uNewCrX;
5977	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_CR2;
5978	rcStrict = VINF_SUCCESS;
5979	break;
5980	}
5981
5982	/*
5983	* CR3 is relatively simple, although AMD and Intel have different
5984	* accounts of how setting reserved bits are handled. We take intel's
5985	* word for the lower bits and AMD's for the high bits (63:52). The
5986	* lower reserved bits are ignored and left alone; OpenBSD 5.8 relies
5987	* on this.
5988	*/
5989	/** @todo Testcase: Setting reserved bits in CR3, especially before
5990	* enabling paging. */
5991	case 3:
5992	{
5993	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5994
5995	/* Bit 63 being clear in the source operand with PCIDE indicates no invalidations are required. */
5996	if ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCIDE)
5997	&& (uNewCrX & RT_BIT_64(63)))
5998	{
5999	/** @todo r=ramshankar: avoiding a TLB flush altogether here causes Windows 10
6000	* SMP(w/o nested-paging) to hang during bootup on Skylake systems, see
6001	* Intel spec. 4.10.4.1 "Operations that Invalidate TLBs and
6002	* Paging-Structure Caches". */
6003	uNewCrX &= ~RT_BIT_64(63);
6004	}
6005
6006	/* Check / mask the value. */
6007	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6008	/* See Intel spec. 27.2.2 "EPT Translation Mechanism" footnote. */
6009	uint64_t const fInvPhysMask = !CPUMIsGuestVmxEptPagingEnabledEx(IEM_GET_CTX(pVCpu))
6010	? (UINT64_MAX << IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth)
6011	: (~X86_CR3_EPT_PAGE_MASK & X86_PAGE_4K_BASE_MASK);
6012	#else
6013	uint64_t const fInvPhysMask = UINT64_C(0xfff0000000000000);
6014	#endif
6015	if (uNewCrX & fInvPhysMask)
6016	{
6017	/** @todo Should we raise this only for 64-bit mode like Intel claims? AMD is
6018	* very vague in this area. As mentioned above, need testcase on real
6019	* hardware... Sigh. */
6020	Log(("Trying to load CR3 with invalid high bits set: %#llx\n", uNewCrX));
6021	return iemRaiseGeneralProtectionFault0(pVCpu);
6022	}
6023
6024	uint64_t fValid;
6025	if ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
6026	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME))
6027	{
6028	/** @todo Redundant? This value has already been validated above. */
6029	fValid = UINT64_C(0x000fffffffffffff);
6030	}
6031	else
6032	fValid = UINT64_C(0xffffffff);
6033	if (uNewCrX & ~fValid)
6034	{
6035	Log(("Automatically clearing reserved MBZ bits in CR3 load: NewCR3=%#llx ClearedBits=%#llx\n",
6036	uNewCrX, uNewCrX & ~fValid));
6037	uNewCrX &= fValid;
6038	}
6039
6040	if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /cr/ 3))
6041	{
6042	Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
6043	IEM_SVM_UPDATE_NRIP(pVCpu);
6044	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR3, enmAccessCrX, iGReg);
6045	}
6046
6047	/* Inform PGM. */
6048	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG)
6049	{
6050	if ( !CPUMIsGuestInPAEModeEx(IEM_GET_CTX(pVCpu))
6051	\|\| CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
6052	{ /* likely */ }
6053	else
6054	{
6055	Assert(enmAccessCrX == IEMACCESSCRX_MOV_CRX);
6056	IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, uNewCrX);
6057	}
6058	rc = PGMFlushTLB(pVCpu, uNewCrX, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
6059	AssertRCReturn(rc, rc);
6060	/* ignore informational status codes */
6061	}
6062
6063	/* Make the change. */
6064	rc = CPUMSetGuestCR3(pVCpu, uNewCrX);
6065	AssertRCSuccessReturn(rc, rc);
6066
6067	rcStrict = VINF_SUCCESS;
6068	break;
6069	}
6070
6071	/*
6072	* CR4 is a bit more tedious as there are bits which cannot be cleared
6073	* under some circumstances and such.
6074	*/
6075	case 4:
6076	{
6077	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6078	uint64_t const uOldCrX = pVCpu->cpum.GstCtx.cr4;
6079
6080	/* Reserved bits. */
6081	uint32_t const fValid = CPUMGetGuestCR4ValidMask(pVCpu->CTX_SUFF(pVM));
6082	if (uNewCrX & ~(uint64_t)fValid)
6083	{
6084	Log(("Trying to set reserved CR4 bits: NewCR4=%#llx InvalidBits=%#llx\n", uNewCrX, uNewCrX & ~(uint64_t)fValid));
6085	return iemRaiseGeneralProtectionFault0(pVCpu);
6086	}
6087
6088	bool const fPcide = !(uOldCrX & X86_CR4_PCIDE) && (uNewCrX & X86_CR4_PCIDE);
6089	bool const fLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu));
6090
6091	/* PCIDE check. */
6092	if ( fPcide
6093	&& ( !fLongMode
6094	\|\| (pVCpu->cpum.GstCtx.cr3 & UINT64_C(0xfff))))
6095	{
6096	Log(("Trying to set PCIDE with invalid PCID or outside long mode. Pcid=%#x\n", (pVCpu->cpum.GstCtx.cr3 & UINT64_C(0xfff))));
6097	return iemRaiseGeneralProtectionFault0(pVCpu);
6098	}
6099
6100	/* PAE check. */
6101	if ( fLongMode
6102	&& (uOldCrX & X86_CR4_PAE)
6103	&& !(uNewCrX & X86_CR4_PAE))
6104	{
6105	Log(("Trying to set clear CR4.PAE while long mode is active\n"));
6106	return iemRaiseGeneralProtectionFault0(pVCpu);
6107	}
6108
6109	if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /cr/ 4))
6110	{
6111	Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
6112	IEM_SVM_UPDATE_NRIP(pVCpu);
6113	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR4, enmAccessCrX, iGReg);
6114	}
6115
6116	/* Check for bits that must remain set or cleared in VMX operation,
6117	see Intel spec. 23.8 "Restrictions on VMX operation". */
6118	if (IEM_VMX_IS_ROOT_MODE(pVCpu))
6119	{
6120	uint64_t const uCr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
6121	if ((uNewCrX & uCr4Fixed0) != uCr4Fixed0)
6122	{
6123	Log(("Trying to clear reserved CR4 bits in VMX operation: NewCr4=%#llx MB1=%#llx\n", uNewCrX, uCr4Fixed0));
6124	return iemRaiseGeneralProtectionFault0(pVCpu);
6125	}
6126
6127	uint64_t const uCr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
6128	if (uNewCrX & ~uCr4Fixed1)
6129	{
6130	Log(("Trying to set reserved CR4 bits in VMX operation: NewCr4=%#llx MB0=%#llx\n", uNewCrX, uCr4Fixed1));
6131	return iemRaiseGeneralProtectionFault0(pVCpu);
6132	}
6133	}
6134
6135	/*
6136	* Notify PGM.
6137	*/
6138	if ((uNewCrX ^ uOldCrX) & (X86_CR4_PSE \| X86_CR4_PAE \| X86_CR4_PGE \| X86_CR4_PCIDE /* \| X86_CR4_SMEP */))
6139	{
6140	if ( !CPUMIsPaePagingEnabled(pVCpu->cpum.GstCtx.cr0, uNewCrX, pVCpu->cpum.GstCtx.msrEFER)
6141	\|\| CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
6142	{ /* likely */ }
6143	else
6144	{
6145	Assert(enmAccessCrX == IEMACCESSCRX_MOV_CRX);
6146	IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, pVCpu->cpum.GstCtx.cr3);
6147	}
6148	rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
6149	AssertRCReturn(rc, rc);
6150	/* ignore informational status codes */
6151	}
6152
6153	/*
6154	* Change it.
6155	*/
6156	rc = CPUMSetGuestCR4(pVCpu, uNewCrX);
6157	AssertRCSuccessReturn(rc, rc);
6158	Assert(pVCpu->cpum.GstCtx.cr4 == uNewCrX);
6159
6160	rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
6161	false /* fForce */);
6162	break;
6163	}
6164
6165	/*
6166	* CR8 maps to the APIC TPR.
6167	*/
6168	case 8:
6169	{
6170	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
6171	if (uNewCrX & ~(uint64_t)0xf)
6172	{
6173	Log(("Trying to set reserved CR8 bits (%#RX64)\n", uNewCrX));
6174	return iemRaiseGeneralProtectionFault0(pVCpu);
6175	}
6176
6177	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6178	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6179	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_USE_TPR_SHADOW))
6180	{
6181	/*
6182	* If the Mov-to-CR8 doesn't cause a VM-exit, bits 0:3 of the source operand
6183	* is copied to bits 7:4 of the VTPR. Bits 0:3 and bits 31:8 of the VTPR are
6184	* cleared. Following this the processor performs TPR virtualization.
6185	*
6186	* However, we should not perform TPR virtualization immediately here but
6187	* after this instruction has completed.
6188	*
6189	* See Intel spec. 29.3 "Virtualizing CR8-based TPR Accesses"
6190	* See Intel spec. 27.1 "Architectural State Before A VM-exit"
6191	*/
6192	uint32_t const uTpr = (uNewCrX & 0xf) << 4;
6193	Log(("iemCImpl_load_Cr%#x: Virtualizing TPR (%#x) write\n", iCrReg, uTpr));
6194	iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_TPR, uTpr);
6195	iemVmxVirtApicSetPendingWrite(pVCpu, XAPIC_OFF_TPR);
6196	rcStrict = VINF_SUCCESS;
6197	break;
6198	}
6199	#endif
6200
6201	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
6202	if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
6203	{
6204	if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /cr/ 8))
6205	{
6206	Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
6207	IEM_SVM_UPDATE_NRIP(pVCpu);
6208	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR8, enmAccessCrX, iGReg);
6209	}
6210
6211	pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.IntCtrl.n.u8VTPR = uNewCrX;
6212	if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, IEM_GET_CTX(pVCpu)))
6213	{
6214	rcStrict = VINF_SUCCESS;
6215	break;
6216	}
6217	}
6218	#endif
6219	uint8_t const u8Tpr = (uint8_t)uNewCrX << 4;
6220	APICSetTpr(pVCpu, u8Tpr);
6221	rcStrict = VINF_SUCCESS;
6222	break;
6223	}
6224
6225	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
6226	}
6227
6228	/*
6229	* Advance the RIP on success.
6230	*/
6231	if (RT_SUCCESS(rcStrict))
6232	{
6233	if (rcStrict != VINF_SUCCESS)
6234	iemSetPassUpStatus(pVCpu, rcStrict);
6235	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6236	}
6237
6238	return rcStrict;
6239	}
6240
6241
6242	/**
6243	* Implements mov CRx,GReg.
6244	*
6245	* @param iCrReg The CRx register to write (valid).
6246	* @param iGReg The general register to load the CRx value from.
6247	*/
6248	IEM_CIMPL_DEF_2(iemCImpl_mov_Cd_Rd, uint8_t, iCrReg, uint8_t, iGReg)
6249	{
6250	if (pVCpu->iem.s.uCpl != 0)
6251	return iemRaiseGeneralProtectionFault0(pVCpu);
6252	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6253
6254	/*
6255	* Read the new value from the source register and call common worker.
6256	*/
6257	uint64_t uNewCrX;
6258	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6259	uNewCrX = iemGRegFetchU64(pVCpu, iGReg);
6260	else
6261	uNewCrX = iemGRegFetchU32(pVCpu, iGReg);
6262
6263	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6264	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6265	{
6266	VBOXSTRICTRC rcStrict = VINF_VMX_INTERCEPT_NOT_ACTIVE;
6267	switch (iCrReg)
6268	{
6269	case 0:
6270	case 4: rcStrict = iemVmxVmexitInstrMovToCr0Cr4(pVCpu, iCrReg, &uNewCrX, iGReg, cbInstr); break;
6271	case 3: rcStrict = iemVmxVmexitInstrMovToCr3(pVCpu, uNewCrX, iGReg, cbInstr); break;
6272	case 8: rcStrict = iemVmxVmexitInstrMovToCr8(pVCpu, iGReg, cbInstr); break;
6273	}
6274	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6275	return rcStrict;
6276	}
6277	#endif
6278
6279	return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, iCrReg, uNewCrX, IEMACCESSCRX_MOV_CRX, iGReg);
6280	}
6281
6282
6283	/**
6284	* Implements 'LMSW r/m16'
6285	*
6286	* @param u16NewMsw The new value.
6287	* @param GCPtrEffDst The guest-linear address of the source operand in case
6288	* of a memory operand. For register operand, pass
6289	* NIL_RTGCPTR.
6290	*/
6291	IEM_CIMPL_DEF_2(iemCImpl_lmsw, uint16_t, u16NewMsw, RTGCPTR, GCPtrEffDst)
6292	{
6293	if (pVCpu->iem.s.uCpl != 0)
6294	return iemRaiseGeneralProtectionFault0(pVCpu);
6295	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6296	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
6297
6298	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6299	/* Check nested-guest VMX intercept and get updated MSW if there's no VM-exit. */
6300	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6301	{
6302	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrLmsw(pVCpu, pVCpu->cpum.GstCtx.cr0, &u16NewMsw, GCPtrEffDst, cbInstr);
6303	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6304	return rcStrict;
6305	}
6306	#else
6307	RT_NOREF_PV(GCPtrEffDst);
6308	#endif
6309
6310	/*
6311	* Compose the new CR0 value and call common worker.
6312	*/
6313	uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0 & ~(X86_CR0_MP \| X86_CR0_EM \| X86_CR0_TS);
6314	uNewCr0 \|= u16NewMsw & (X86_CR0_PE \| X86_CR0_MP \| X86_CR0_EM \| X86_CR0_TS);
6315	return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, /cr/ 0, uNewCr0, IEMACCESSCRX_LMSW, UINT8_MAX /* iGReg */);
6316	}
6317
6318
6319	/**
6320	* Implements 'CLTS'.
6321	*/
6322	IEM_CIMPL_DEF_0(iemCImpl_clts)
6323	{
6324	if (pVCpu->iem.s.uCpl != 0)
6325	return iemRaiseGeneralProtectionFault0(pVCpu);
6326
6327	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
6328	uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0;
6329	uNewCr0 &= ~X86_CR0_TS;
6330
6331	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6332	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6333	{
6334	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrClts(pVCpu, cbInstr);
6335	if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR)
6336	uNewCr0 \|= (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS);
6337	else if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6338	return rcStrict;
6339	}
6340	#endif
6341
6342	return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, /cr/ 0, uNewCr0, IEMACCESSCRX_CLTS, UINT8_MAX /* iGReg */);
6343	}
6344
6345
6346	/**
6347	* Implements mov GReg,DRx.
6348	*
6349	* @param iGReg The general register to store the DRx value in.
6350	* @param iDrReg The DRx register to read (0-7).
6351	*/
6352	IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Dd, uint8_t, iGReg, uint8_t, iDrReg)
6353	{
6354	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6355	/*
6356	* Check nested-guest VMX intercept.
6357	* Unlike most other intercepts, the Mov DRx intercept takes preceedence
6358	* over CPL and CR4.DE and even DR4/DR5 checks.
6359	*
6360	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
6361	*/
6362	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6363	{
6364	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovDrX(pVCpu, VMXINSTRID_MOV_FROM_DRX, iDrReg, iGReg, cbInstr);
6365	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6366	return rcStrict;
6367	}
6368	#endif
6369
6370	/*
6371	* Check preconditions.
6372	*/
6373	/* Raise GPs. */
6374	if (pVCpu->iem.s.uCpl != 0)
6375	return iemRaiseGeneralProtectionFault0(pVCpu);
6376	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6377	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7 \| CPUMCTX_EXTRN_CR0);
6378
6379	if ( (iDrReg == 4 \|\| iDrReg == 5)
6380	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE) )
6381	{
6382	Log(("mov r%u,dr%u: CR4.DE=1 -> #GP(0)\n", iGReg, iDrReg));
6383	return iemRaiseGeneralProtectionFault0(pVCpu);
6384	}
6385
6386	/* Raise #DB if general access detect is enabled. */
6387	if (pVCpu->cpum.GstCtx.dr[7] & X86_DR7_GD)
6388	{
6389	Log(("mov r%u,dr%u: DR7.GD=1 -> #DB\n", iGReg, iDrReg));
6390	return iemRaiseDebugException(pVCpu);
6391	}
6392
6393	/*
6394	* Read the debug register and store it in the specified general register.
6395	*/
6396	uint64_t drX;
6397	switch (iDrReg)
6398	{
6399	case 0:
6400	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
6401	drX = pVCpu->cpum.GstCtx.dr[0];
6402	break;
6403	case 1:
6404	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
6405	drX = pVCpu->cpum.GstCtx.dr[1];
6406	break;
6407	case 2:
6408	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
6409	drX = pVCpu->cpum.GstCtx.dr[2];
6410	break;
6411	case 3:
6412	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
6413	drX = pVCpu->cpum.GstCtx.dr[3];
6414	break;
6415	case 6:
6416	case 4:
6417	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
6418	drX = pVCpu->cpum.GstCtx.dr[6];
6419	drX \|= X86_DR6_RA1_MASK;
6420	drX &= ~X86_DR6_RAZ_MASK;
6421	break;
6422	case 7:
6423	case 5:
6424	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
6425	drX = pVCpu->cpum.GstCtx.dr[7];
6426	drX \|=X86_DR7_RA1_MASK;
6427	drX &= ~X86_DR7_RAZ_MASK;
6428	break;
6429	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* caller checks */
6430	}
6431
6432	/** @todo SVM nested-guest intercept for DR8-DR15? */
6433	/*
6434	* Check for any SVM nested-guest intercepts for the DRx read.
6435	*/
6436	if (IEM_SVM_IS_READ_DR_INTERCEPT_SET(pVCpu, iDrReg))
6437	{
6438	Log(("mov r%u,dr%u: Guest intercept -> #VMEXIT\n", iGReg, iDrReg));
6439	IEM_SVM_UPDATE_NRIP(pVCpu);
6440	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_READ_DR0 + (iDrReg & 0xf),
6441	IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? (iGReg & 7) : 0, 0 /* uExitInfo2 */);
6442	}
6443
6444	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6445	(uint64_t )iemGRegRef(pVCpu, iGReg) = drX;
6446	else
6447	(uint64_t )iemGRegRef(pVCpu, iGReg) = (uint32_t)drX;
6448
6449	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6450	}
6451
6452
6453	/**
6454	* Implements mov DRx,GReg.
6455	*
6456	* @param iDrReg The DRx register to write (valid).
6457	* @param iGReg The general register to load the DRx value from.
6458	*/
6459	IEM_CIMPL_DEF_2(iemCImpl_mov_Dd_Rd, uint8_t, iDrReg, uint8_t, iGReg)
6460	{
6461	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6462	/*
6463	* Check nested-guest VMX intercept.
6464	* Unlike most other intercepts, the Mov DRx intercept takes preceedence
6465	* over CPL and CR4.DE and even DR4/DR5 checks.
6466	*
6467	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
6468	*/
6469	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6470	{
6471	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovDrX(pVCpu, VMXINSTRID_MOV_TO_DRX, iDrReg, iGReg, cbInstr);
6472	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6473	return rcStrict;
6474	}
6475	#endif
6476
6477	/*
6478	* Check preconditions.
6479	*/
6480	if (pVCpu->iem.s.uCpl != 0)
6481	return iemRaiseGeneralProtectionFault0(pVCpu);
6482	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6483	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7 \| CPUMCTX_EXTRN_CR4);
6484
6485	if (iDrReg == 4 \|\| iDrReg == 5)
6486	{
6487	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE)
6488	{
6489	Log(("mov dr%u,r%u: CR4.DE=1 -> #GP(0)\n", iDrReg, iGReg));
6490	return iemRaiseGeneralProtectionFault0(pVCpu);
6491	}
6492	iDrReg += 2;
6493	}
6494
6495	/* Raise #DB if general access detect is enabled. */
6496	/** @todo is \#DB/DR7.GD raised before any reserved high bits in DR7/DR6
6497	* \#GP? */
6498	if (pVCpu->cpum.GstCtx.dr[7] & X86_DR7_GD)
6499	{
6500	Log(("mov dr%u,r%u: DR7.GD=1 -> #DB\n", iDrReg, iGReg));
6501	return iemRaiseDebugException(pVCpu);
6502	}
6503
6504	/*
6505	* Read the new value from the source register.
6506	*/
6507	uint64_t uNewDrX;
6508	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6509	uNewDrX = iemGRegFetchU64(pVCpu, iGReg);
6510	else
6511	uNewDrX = iemGRegFetchU32(pVCpu, iGReg);
6512
6513	/*
6514	* Adjust it.
6515	*/
6516	switch (iDrReg)
6517	{
6518	case 0:
6519	case 1:
6520	case 2:
6521	case 3:
6522	/* nothing to adjust */
6523	break;
6524
6525	case 6:
6526	if (uNewDrX & X86_DR6_MBZ_MASK)
6527	{
6528	Log(("mov dr%u,%#llx: DR6 high bits are not zero -> #GP(0)\n", iDrReg, uNewDrX));
6529	return iemRaiseGeneralProtectionFault0(pVCpu);
6530	}
6531	uNewDrX \|= X86_DR6_RA1_MASK;
6532	uNewDrX &= ~X86_DR6_RAZ_MASK;
6533	break;
6534
6535	case 7:
6536	if (uNewDrX & X86_DR7_MBZ_MASK)
6537	{
6538	Log(("mov dr%u,%#llx: DR7 high bits are not zero -> #GP(0)\n", iDrReg, uNewDrX));
6539	return iemRaiseGeneralProtectionFault0(pVCpu);
6540	}
6541	uNewDrX \|= X86_DR7_RA1_MASK;
6542	uNewDrX &= ~X86_DR7_RAZ_MASK;
6543	break;
6544
6545	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6546	}
6547
6548	/** @todo SVM nested-guest intercept for DR8-DR15? */
6549	/*
6550	* Check for any SVM nested-guest intercepts for the DRx write.
6551	*/
6552	if (IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(pVCpu, iDrReg))
6553	{
6554	Log2(("mov dr%u,r%u: Guest intercept -> #VMEXIT\n", iDrReg, iGReg));
6555	IEM_SVM_UPDATE_NRIP(pVCpu);
6556	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_DR0 + (iDrReg & 0xf),
6557	IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? (iGReg & 7) : 0, 0 /* uExitInfo2 */);
6558	}
6559
6560	/*
6561	* Do the actual setting.
6562	*/
6563	if (iDrReg < 4)
6564	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
6565	else if (iDrReg == 6)
6566	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
6567
6568	int rc = CPUMSetGuestDRx(pVCpu, iDrReg, uNewDrX);
6569	AssertRCSuccessReturn(rc, RT_SUCCESS_NP(rc) ? VERR_IEM_IPE_1 : rc);
6570
6571	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6572	}
6573
6574
6575	/**
6576	* Implements mov GReg,TRx.
6577	*
6578	* @param iGReg The general register to store the
6579	* TRx value in.
6580	* @param iTrReg The TRx register to read (6/7).
6581	*/
6582	IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Td, uint8_t, iGReg, uint8_t, iTrReg)
6583	{
6584	/*
6585	* Check preconditions. NB: This instruction is 386/486 only.
6586	*/
6587
6588	/* Raise GPs. */
6589	if (pVCpu->iem.s.uCpl != 0)
6590	return iemRaiseGeneralProtectionFault0(pVCpu);
6591	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6592
6593	if (iTrReg < 6 \|\| iTrReg > 7)
6594	{
6595	/** @todo Do Intel CPUs reject this or are the TRs aliased? */
6596	Log(("mov r%u,tr%u: invalid register -> #GP(0)\n", iGReg, iTrReg));
6597	return iemRaiseGeneralProtectionFault0(pVCpu);
6598	}
6599
6600	/*
6601	* Read the test register and store it in the specified general register.
6602	* This is currently a dummy implementation that only exists to satisfy
6603	* old debuggers like WDEB386 or OS/2 KDB which unconditionally read the
6604	* TR6/TR7 registers. Software which actually depends on the TR values
6605	* (different on 386/486) is exceedingly rare.
6606	*/
6607	uint64_t trX;
6608	switch (iTrReg)
6609	{
6610	case 6:
6611	trX = 0; /* Currently a dummy. */
6612	break;
6613	case 7:
6614	trX = 0; /* Currently a dummy. */
6615	break;
6616	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
6617	}
6618
6619	(uint64_t )iemGRegRef(pVCpu, iGReg) = (uint32_t)trX;
6620
6621	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6622	}
6623
6624
6625	/**
6626	* Implements mov TRx,GReg.
6627	*
6628	* @param iTrReg The TRx register to write (valid).
6629	* @param iGReg The general register to load the TRx
6630	* value from.
6631	*/
6632	IEM_CIMPL_DEF_2(iemCImpl_mov_Td_Rd, uint8_t, iTrReg, uint8_t, iGReg)
6633	{
6634	/*
6635	* Check preconditions. NB: This instruction is 386/486 only.
6636	*/
6637
6638	/* Raise GPs. */
6639	if (pVCpu->iem.s.uCpl != 0)
6640	return iemRaiseGeneralProtectionFault0(pVCpu);
6641	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6642
6643	if (iTrReg < 6 \|\| iTrReg > 7)
6644	{
6645	/** @todo Do Intel CPUs reject this or are the TRs aliased? */
6646	Log(("mov r%u,tr%u: invalid register -> #GP(0)\n", iGReg, iTrReg));
6647	return iemRaiseGeneralProtectionFault0(pVCpu);
6648	}
6649
6650	/*
6651	* Read the new value from the source register.
6652	*/
6653	uint64_t uNewTrX;
6654	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6655	uNewTrX = iemGRegFetchU64(pVCpu, iGReg);
6656	else
6657	uNewTrX = iemGRegFetchU32(pVCpu, iGReg);
6658
6659	/*
6660	* Here we would do the actual setting if this weren't a dummy implementation.
6661	* This is currently a dummy implementation that only exists to prevent
6662	* old debuggers like WDEB386 or OS/2 KDB from crashing.
6663	*/
6664	RT_NOREF(uNewTrX);
6665
6666	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6667	}
6668
6669
6670	/**
6671	* Implements 'INVLPG m'.
6672	*
6673	* @param GCPtrPage The effective address of the page to invalidate.
6674	* @remarks Updates the RIP.
6675	*/
6676	IEM_CIMPL_DEF_1(iemCImpl_invlpg, RTGCPTR, GCPtrPage)
6677	{
6678	/* ring-0 only. */
6679	if (pVCpu->iem.s.uCpl != 0)
6680	return iemRaiseGeneralProtectionFault0(pVCpu);
6681	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6682	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
6683
6684	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6685	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6686	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INVLPG_EXIT))
6687	{
6688	Log(("invlpg: Guest intercept (%RGp) -> VM-exit\n", GCPtrPage));
6689	return iemVmxVmexitInstrInvlpg(pVCpu, GCPtrPage, cbInstr);
6690	}
6691	#endif
6692
6693	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INVLPG))
6694	{
6695	Log(("invlpg: Guest intercept (%RGp) -> #VMEXIT\n", GCPtrPage));
6696	IEM_SVM_UPDATE_NRIP(pVCpu);
6697	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_INVLPG,
6698	IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? GCPtrPage : 0, 0 /* uExitInfo2 */);
6699	}
6700
6701	int rc = PGMInvalidatePage(pVCpu, GCPtrPage);
6702	if (rc == VINF_SUCCESS)
6703	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6704	if (rc == VINF_PGM_SYNC_CR3)
6705	{
6706	iemSetPassUpStatus(pVCpu, rc);
6707	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6708	}
6709
6710	AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
6711	Log(("PGMInvalidatePage(%RGv) -> %Rrc\n", GCPtrPage, rc));
6712	return rc;
6713	}
6714
6715
6716	/**
6717	* Implements INVPCID.
6718	*
6719	* @param iEffSeg The segment of the invpcid descriptor.
6720	* @param GCPtrInvpcidDesc The address of invpcid descriptor.
6721	* @param uInvpcidType The invalidation type.
6722	* @remarks Updates the RIP.
6723	*/
6724	IEM_CIMPL_DEF_3(iemCImpl_invpcid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvpcidDesc, uint64_t, uInvpcidType)
6725	{
6726	/*
6727	* Check preconditions.
6728	*/
6729	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fInvpcid)
6730	return iemRaiseUndefinedOpcode(pVCpu);
6731
6732	/* When in VMX non-root mode and INVPCID is not enabled, it results in #UD. */
6733	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6734	&& !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_INVPCID))
6735	{
6736	Log(("invpcid: Not enabled for nested-guest execution -> #UD\n"));
6737	return iemRaiseUndefinedOpcode(pVCpu);
6738	}
6739
6740	if (pVCpu->iem.s.uCpl != 0)
6741	{
6742	Log(("invpcid: CPL != 0 -> #GP(0)\n"));
6743	return iemRaiseGeneralProtectionFault0(pVCpu);
6744	}
6745
6746	if (IEM_IS_V86_MODE(pVCpu))
6747	{
6748	Log(("invpcid: v8086 mode -> #GP(0)\n"));
6749	return iemRaiseGeneralProtectionFault0(pVCpu);
6750	}
6751
6752	/*
6753	* Check nested-guest intercept.
6754	*
6755	* INVPCID causes a VM-exit if "enable INVPCID" and "INVLPG exiting" are
6756	* both set. We have already checked the former earlier in this function.
6757	*
6758	* CPL and virtual-8086 mode checks take priority over this VM-exit.
6759	* See Intel spec. "25.1.1 Relative Priority of Faults and VM Exits".
6760	*/
6761	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6762	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INVLPG_EXIT))
6763	{
6764	Log(("invpcid: Guest intercept -> #VM-exit\n"));
6765	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_INVPCID, VMXINSTRID_NONE, cbInstr);
6766	}
6767
6768	if (uInvpcidType > X86_INVPCID_TYPE_MAX_VALID)
6769	{
6770	Log(("invpcid: invalid/unrecognized invpcid type %#RX64 -> #GP(0)\n", uInvpcidType));
6771	return iemRaiseGeneralProtectionFault0(pVCpu);
6772	}
6773	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
6774
6775	/*
6776	* Fetch the invpcid descriptor from guest memory.
6777	*/
6778	RTUINT128U uDesc;
6779	VBOXSTRICTRC rcStrict = iemMemFetchDataU128(pVCpu, &uDesc, iEffSeg, GCPtrInvpcidDesc);
6780	if (rcStrict == VINF_SUCCESS)
6781	{
6782	/*
6783	* Validate the descriptor.
6784	*/
6785	if (uDesc.s.Lo > 0xfff)
6786	{
6787	Log(("invpcid: reserved bits set in invpcid descriptor %#RX64 -> #GP(0)\n", uDesc.s.Lo));
6788	return iemRaiseGeneralProtectionFault0(pVCpu);
6789	}
6790
6791	RTGCUINTPTR64 const GCPtrInvAddr = uDesc.s.Hi;
6792	uint8_t const uPcid = uDesc.s.Lo & UINT64_C(0xfff);
6793	uint32_t const uCr4 = pVCpu->cpum.GstCtx.cr4;
6794	uint64_t const uCr3 = pVCpu->cpum.GstCtx.cr3;
6795	switch (uInvpcidType)
6796	{
6797	case X86_INVPCID_TYPE_INDV_ADDR:
6798	{
6799	if (!IEM_IS_CANONICAL(GCPtrInvAddr))
6800	{
6801	Log(("invpcid: invalidation address %#RGP is not canonical -> #GP(0)\n", GCPtrInvAddr));
6802	return iemRaiseGeneralProtectionFault0(pVCpu);
6803	}
6804	if ( !(uCr4 & X86_CR4_PCIDE)
6805	&& uPcid != 0)
6806	{
6807	Log(("invpcid: invalid pcid %#x\n", uPcid));
6808	return iemRaiseGeneralProtectionFault0(pVCpu);
6809	}
6810
6811	/* Invalidate mappings for the linear address tagged with PCID except global translations. */
6812	PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
6813	break;
6814	}
6815
6816	case X86_INVPCID_TYPE_SINGLE_CONTEXT:
6817	{
6818	if ( !(uCr4 & X86_CR4_PCIDE)
6819	&& uPcid != 0)
6820	{
6821	Log(("invpcid: invalid pcid %#x\n", uPcid));
6822	return iemRaiseGeneralProtectionFault0(pVCpu);
6823	}
6824	/* Invalidate all mappings associated with PCID except global translations. */
6825	PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
6826	break;
6827	}
6828
6829	case X86_INVPCID_TYPE_ALL_CONTEXT_INCL_GLOBAL:
6830	{
6831	PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
6832	break;
6833	}
6834
6835	case X86_INVPCID_TYPE_ALL_CONTEXT_EXCL_GLOBAL:
6836	{
6837	PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
6838	break;
6839	}
6840	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6841	}
6842	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6843	}
6844	return rcStrict;
6845	}
6846
6847
6848	/**
6849	* Implements INVD.
6850	*/
6851	IEM_CIMPL_DEF_0(iemCImpl_invd)
6852	{
6853	if (pVCpu->iem.s.uCpl != 0)
6854	{
6855	Log(("invd: CPL != 0 -> #GP(0)\n"));
6856	return iemRaiseGeneralProtectionFault0(pVCpu);
6857	}
6858
6859	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6860	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_INVD, cbInstr);
6861
6862	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_INVD, SVM_EXIT_INVD, 0, 0);
6863
6864	/* We currently take no action here. */
6865	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6866	}
6867
6868
6869	/**
6870	* Implements WBINVD.
6871	*/
6872	IEM_CIMPL_DEF_0(iemCImpl_wbinvd)
6873	{
6874	if (pVCpu->iem.s.uCpl != 0)
6875	{
6876	Log(("wbinvd: CPL != 0 -> #GP(0)\n"));
6877	return iemRaiseGeneralProtectionFault0(pVCpu);
6878	}
6879
6880	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6881	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_WBINVD, cbInstr);
6882
6883	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_WBINVD, SVM_EXIT_WBINVD, 0, 0);
6884
6885	/* We currently take no action here. */
6886	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6887	}
6888
6889
6890	/** Opcode 0x0f 0xaa. */
6891	IEM_CIMPL_DEF_0(iemCImpl_rsm)
6892	{
6893	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_RSM, SVM_EXIT_RSM, 0, 0);
6894	NOREF(cbInstr);
6895	return iemRaiseUndefinedOpcode(pVCpu);
6896	}
6897
6898
6899	/**
6900	* Implements RDTSC.
6901	*/
6902	IEM_CIMPL_DEF_0(iemCImpl_rdtsc)
6903	{
6904	/*
6905	* Check preconditions.
6906	*/
6907	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fTsc)
6908	return iemRaiseUndefinedOpcode(pVCpu);
6909
6910	if (pVCpu->iem.s.uCpl != 0)
6911	{
6912	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6913	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_TSD)
6914	{
6915	Log(("rdtsc: CR4.TSD and CPL=%u -> #GP(0)\n", pVCpu->iem.s.uCpl));
6916	return iemRaiseGeneralProtectionFault0(pVCpu);
6917	}
6918	}
6919
6920	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6921	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDTSC_EXIT))
6922	{
6923	Log(("rdtsc: Guest intercept -> VM-exit\n"));
6924	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDTSC, cbInstr);
6925	}
6926
6927	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDTSC))
6928	{
6929	Log(("rdtsc: Guest intercept -> #VMEXIT\n"));
6930	IEM_SVM_UPDATE_NRIP(pVCpu);
6931	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDTSC, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
6932	}
6933
6934	/*
6935	* Do the job.
6936	*/
6937	uint64_t uTicks = TMCpuTickGet(pVCpu);
6938	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
6939	uTicks = CPUMApplyNestedGuestTscOffset(pVCpu, uTicks);
6940	#endif
6941	pVCpu->cpum.GstCtx.rax = RT_LO_U32(uTicks);
6942	pVCpu->cpum.GstCtx.rdx = RT_HI_U32(uTicks);
6943	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX); /* For IEMExecDecodedRdtsc. */
6944	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6945	}
6946
6947
6948	/**
6949	* Implements RDTSC.
6950	*/
6951	IEM_CIMPL_DEF_0(iemCImpl_rdtscp)
6952	{
6953	/*
6954	* Check preconditions.
6955	*/
6956	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fRdTscP)
6957	return iemRaiseUndefinedOpcode(pVCpu);
6958
6959	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6960	&& !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_RDTSCP))
6961	{
6962	Log(("rdtscp: Not enabled for VMX non-root mode -> #UD\n"));
6963	return iemRaiseUndefinedOpcode(pVCpu);
6964	}
6965
6966	if (pVCpu->iem.s.uCpl != 0)
6967	{
6968	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6969	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_TSD)
6970	{
6971	Log(("rdtscp: CR4.TSD and CPL=%u -> #GP(0)\n", pVCpu->iem.s.uCpl));
6972	return iemRaiseGeneralProtectionFault0(pVCpu);
6973	}
6974	}
6975
6976	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6977	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDTSC_EXIT))
6978	{
6979	Log(("rdtscp: Guest intercept -> VM-exit\n"));
6980	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDTSCP, cbInstr);
6981	}
6982	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDTSCP))
6983	{
6984	Log(("rdtscp: Guest intercept -> #VMEXIT\n"));
6985	IEM_SVM_UPDATE_NRIP(pVCpu);
6986	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDTSCP, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
6987	}
6988
6989	/*
6990	* Do the job.
6991	* Query the MSR first in case of trips to ring-3.
6992	*/
6993	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
6994	VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pVCpu->cpum.GstCtx.rcx);
6995	if (rcStrict == VINF_SUCCESS)
6996	{
6997	/* Low dword of the TSC_AUX msr only. */
6998	pVCpu->cpum.GstCtx.rcx &= UINT32_C(0xffffffff);
6999
7000	uint64_t uTicks = TMCpuTickGet(pVCpu);
7001	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
7002	uTicks = CPUMApplyNestedGuestTscOffset(pVCpu, uTicks);
7003	#endif
7004	pVCpu->cpum.GstCtx.rax = RT_LO_U32(uTicks);
7005	pVCpu->cpum.GstCtx.rdx = RT_HI_U32(uTicks);
7006	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX); /* For IEMExecDecodedRdtscp. */
7007	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7008	}
7009	return rcStrict;
7010	}
7011
7012
7013	/**
7014	* Implements RDPMC.
7015	*/
7016	IEM_CIMPL_DEF_0(iemCImpl_rdpmc)
7017	{
7018	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7019
7020	if ( pVCpu->iem.s.uCpl != 0
7021	&& !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCE))
7022	return iemRaiseGeneralProtectionFault0(pVCpu);
7023
7024	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7025	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDPMC_EXIT))
7026	{
7027	Log(("rdpmc: Guest intercept -> VM-exit\n"));
7028	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDPMC, cbInstr);
7029	}
7030
7031	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDPMC))
7032	{
7033	Log(("rdpmc: Guest intercept -> #VMEXIT\n"));
7034	IEM_SVM_UPDATE_NRIP(pVCpu);
7035	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDPMC, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7036	}
7037
7038	/** @todo Emulate performance counters, for now just return 0. */
7039	pVCpu->cpum.GstCtx.rax = 0;
7040	pVCpu->cpum.GstCtx.rdx = 0;
7041	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX);
7042	/** @todo We should trigger a \#GP here if the CPU doesn't support the index in
7043	* ecx but see @bugref{3472}! */
7044
7045	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7046	}
7047
7048
7049	/**
7050	* Implements RDMSR.
7051	*/
7052	IEM_CIMPL_DEF_0(iemCImpl_rdmsr)
7053	{
7054	/*
7055	* Check preconditions.
7056	*/
7057	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMsr)
7058	return iemRaiseUndefinedOpcode(pVCpu);
7059	if (pVCpu->iem.s.uCpl != 0)
7060	return iemRaiseGeneralProtectionFault0(pVCpu);
7061
7062	/*
7063	* Check nested-guest intercepts.
7064	*/
7065	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7066	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7067	{
7068	if (iemVmxIsRdmsrWrmsrInterceptSet(pVCpu, VMX_EXIT_RDMSR, pVCpu->cpum.GstCtx.ecx))
7069	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDMSR, cbInstr);
7070	}
7071	#endif
7072
7073	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7074	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT))
7075	{
7076	VBOXSTRICTRC rcStrict = iemSvmHandleMsrIntercept(pVCpu, pVCpu->cpum.GstCtx.ecx, false /* fWrite */);
7077	if (rcStrict == VINF_SVM_VMEXIT)
7078	return VINF_SUCCESS;
7079	if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
7080	{
7081	Log(("IEM: SVM intercepted rdmsr(%#x) failed. rc=%Rrc\n", pVCpu->cpum.GstCtx.ecx, VBOXSTRICTRC_VAL(rcStrict)));
7082	return rcStrict;
7083	}
7084	}
7085	#endif
7086
7087	/*
7088	* Do the job.
7089	*/
7090	RTUINT64U uValue;
7091	/** @todo make CPUMAllMsrs.cpp import the necessary MSR state. */
7092	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
7093
7094	VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pVCpu->cpum.GstCtx.ecx, &uValue.u);
7095	if (rcStrict == VINF_SUCCESS)
7096	{
7097	pVCpu->cpum.GstCtx.rax = uValue.s.Lo;
7098	pVCpu->cpum.GstCtx.rdx = uValue.s.Hi;
7099	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX);
7100
7101	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7102	}
7103
7104	#ifndef IN_RING3
7105	/* Deferred to ring-3. */
7106	if (rcStrict == VINF_CPUM_R3_MSR_READ)
7107	{
7108	Log(("IEM: rdmsr(%#x) -> ring-3\n", pVCpu->cpum.GstCtx.ecx));
7109	return rcStrict;
7110	}
7111	#endif
7112
7113	/* Often a unimplemented MSR or MSR bit, so worth logging. */
7114	if (pVCpu->iem.s.cLogRelRdMsr < 32)
7115	{
7116	pVCpu->iem.s.cLogRelRdMsr++;
7117	LogRel(("IEM: rdmsr(%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.ecx));
7118	}
7119	else
7120	Log(( "IEM: rdmsr(%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.ecx));
7121	AssertMsgReturn(rcStrict == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IPE_UNEXPECTED_STATUS);
7122	return iemRaiseGeneralProtectionFault0(pVCpu);
7123	}
7124
7125
7126	/**
7127	* Implements WRMSR.
7128	*/
7129	IEM_CIMPL_DEF_0(iemCImpl_wrmsr)
7130	{
7131	/*
7132	* Check preconditions.
7133	*/
7134	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMsr)
7135	return iemRaiseUndefinedOpcode(pVCpu);
7136	if (pVCpu->iem.s.uCpl != 0)
7137	return iemRaiseGeneralProtectionFault0(pVCpu);
7138
7139	RTUINT64U uValue;
7140	uValue.s.Lo = pVCpu->cpum.GstCtx.eax;
7141	uValue.s.Hi = pVCpu->cpum.GstCtx.edx;
7142
7143	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
7144
7145	/** @todo make CPUMAllMsrs.cpp import the necessary MSR state. */
7146	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
7147
7148	/*
7149	* Check nested-guest intercepts.
7150	*/
7151	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7152	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7153	{
7154	if (iemVmxIsRdmsrWrmsrInterceptSet(pVCpu, VMX_EXIT_WRMSR, idMsr))
7155	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_WRMSR, cbInstr);
7156	}
7157	#endif
7158
7159	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7160	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT))
7161	{
7162	VBOXSTRICTRC rcStrict = iemSvmHandleMsrIntercept(pVCpu, idMsr, true /* fWrite */);
7163	if (rcStrict == VINF_SVM_VMEXIT)
7164	return VINF_SUCCESS;
7165	if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
7166	{
7167	Log(("IEM: SVM intercepted rdmsr(%#x) failed. rc=%Rrc\n", idMsr, VBOXSTRICTRC_VAL(rcStrict)));
7168	return rcStrict;
7169	}
7170	}
7171	#endif
7172
7173	/*
7174	* Do the job.
7175	*/
7176	VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, idMsr, uValue.u);
7177	if (rcStrict == VINF_SUCCESS)
7178	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7179
7180	#ifndef IN_RING3
7181	/* Deferred to ring-3. */
7182	if (rcStrict == VINF_CPUM_R3_MSR_WRITE)
7183	{
7184	Log(("IEM: wrmsr(%#x) -> ring-3\n", idMsr));
7185	return rcStrict;
7186	}
7187	#endif
7188
7189	/* Often a unimplemented MSR or MSR bit, so worth logging. */
7190	if (pVCpu->iem.s.cLogRelWrMsr < 32)
7191	{
7192	pVCpu->iem.s.cLogRelWrMsr++;
7193	LogRel(("IEM: wrmsr(%#x,%#x`%08x) -> #GP(0)\n", idMsr, uValue.s.Hi, uValue.s.Lo));
7194	}
7195	else
7196	Log(( "IEM: wrmsr(%#x,%#x`%08x) -> #GP(0)\n", idMsr, uValue.s.Hi, uValue.s.Lo));
7197	AssertMsgReturn(rcStrict == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IPE_UNEXPECTED_STATUS);
7198	return iemRaiseGeneralProtectionFault0(pVCpu);
7199	}
7200
7201
7202	/**
7203	* Implements 'IN eAX, port'.
7204	*
7205	* @param u16Port The source port.
7206	* @param fImm Whether the port was specified through an immediate operand
7207	* or the implicit DX register.
7208	* @param cbReg The register size.
7209	*/
7210	IEM_CIMPL_DEF_3(iemCImpl_in, uint16_t, u16Port, bool, fImm, uint8_t, cbReg)
7211	{
7212	/*
7213	* CPL check
7214	*/
7215	VBOXSTRICTRC rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, cbReg);
7216	if (rcStrict != VINF_SUCCESS)
7217	return rcStrict;
7218
7219	/*
7220	* Check VMX nested-guest IO intercept.
7221	*/
7222	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7223	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7224	{
7225	rcStrict = iemVmxVmexitInstrIo(pVCpu, VMXINSTRID_IO_IN, u16Port, fImm, cbReg, cbInstr);
7226	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
7227	return rcStrict;
7228	}
7229	#else
7230	RT_NOREF(fImm);
7231	#endif
7232
7233	/*
7234	* Check SVM nested-guest IO intercept.
7235	*/
7236	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7237	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
7238	{
7239	uint8_t cAddrSizeBits;
7240	switch (pVCpu->iem.s.enmEffAddrMode)
7241	{
7242	case IEMMODE_16BIT: cAddrSizeBits = 16; break;
7243	case IEMMODE_32BIT: cAddrSizeBits = 32; break;
7244	case IEMMODE_64BIT: cAddrSizeBits = 64; break;
7245	IEM_NOT_REACHED_DEFAULT_CASE_RET();
7246	}
7247	rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_IN, cbReg, cAddrSizeBits, 0 /* N/A - iEffSeg */,
7248	false /* fRep /, false / fStrIo */, cbInstr);
7249	if (rcStrict == VINF_SVM_VMEXIT)
7250	return VINF_SUCCESS;
7251	if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
7252	{
7253	Log(("iemCImpl_in: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, cbReg,
7254	VBOXSTRICTRC_VAL(rcStrict)));
7255	return rcStrict;
7256	}
7257	}
7258	#endif
7259
7260	/*
7261	* Perform the I/O.
7262	*/
7263	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
7264	uint32_t u32Value = 0;
7265	rcStrict = IOMIOPortRead(pVM, pVCpu, u16Port, &u32Value, cbReg);
7266	if (IOM_SUCCESS(rcStrict))
7267	{
7268	switch (cbReg)
7269	{
7270	case 1: pVCpu->cpum.GstCtx.al = (uint8_t)u32Value; break;
7271	case 2: pVCpu->cpum.GstCtx.ax = (uint16_t)u32Value; break;
7272	case 4: pVCpu->cpum.GstCtx.rax = u32Value; break;
7273	default: AssertFailedReturn(VERR_IEM_IPE_3);
7274	}
7275
7276	pVCpu->iem.s.cPotentialExits++;
7277	if (rcStrict != VINF_SUCCESS)
7278	iemSetPassUpStatus(pVCpu, rcStrict);
7279	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7280
7281	/*
7282	* Check for I/O breakpoints.
7283	*/
7284	/** @todo this should set a internal flag and be raised by
7285	* iemRegAddToRipAndFinishingClearingRF! */
7286	uint32_t const uDr7 = pVCpu->cpum.GstCtx.dr[7];
7287	if (RT_UNLIKELY( ( ( (uDr7 & X86_DR7_ENABLED_MASK)
7288	&& X86_DR7_ANY_RW_IO(uDr7)
7289	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE))
7290	\|\| DBGFBpIsHwIoArmed(pVM))
7291	&& rcStrict == VINF_SUCCESS))
7292	{
7293	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3 \| CPUMCTX_EXTRN_DR6);
7294	rcStrict = DBGFBpCheckIo(pVM, pVCpu, IEM_GET_CTX(pVCpu), u16Port, cbReg);
7295	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
7296	rcStrict = iemRaiseDebugException(pVCpu);
7297	}
7298	}
7299
7300	return rcStrict;
7301	}
7302
7303
7304	/**
7305	* Implements 'IN eAX, DX'.
7306	*
7307	* @param cbReg The register size.
7308	*/
7309	IEM_CIMPL_DEF_1(iemCImpl_in_eAX_DX, uint8_t, cbReg)
7310	{
7311	return IEM_CIMPL_CALL_3(iemCImpl_in, pVCpu->cpum.GstCtx.dx, false /* fImm */, cbReg);
7312	}
7313
7314
7315	/**
7316	* Implements 'OUT port, eAX'.
7317	*
7318	* @param u16Port The destination port.
7319	* @param fImm Whether the port was specified through an immediate operand
7320	* or the implicit DX register.
7321	* @param cbReg The register size.
7322	*/
7323	IEM_CIMPL_DEF_3(iemCImpl_out, uint16_t, u16Port, bool, fImm, uint8_t, cbReg)
7324	{
7325	/*
7326	* CPL check
7327	*/
7328	VBOXSTRICTRC rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, cbReg);
7329	if (rcStrict != VINF_SUCCESS)
7330	return rcStrict;
7331
7332	/*
7333	* Check VMX nested-guest I/O intercept.
7334	*/
7335	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7336	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7337	{
7338	rcStrict = iemVmxVmexitInstrIo(pVCpu, VMXINSTRID_IO_OUT, u16Port, fImm, cbReg, cbInstr);
7339	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
7340	return rcStrict;
7341	}
7342	#else
7343	RT_NOREF(fImm);
7344	#endif
7345
7346	/*
7347	* Check SVM nested-guest I/O intercept.
7348	*/
7349	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7350	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
7351	{
7352	uint8_t cAddrSizeBits;
7353	switch (pVCpu->iem.s.enmEffAddrMode)
7354	{
7355	case IEMMODE_16BIT: cAddrSizeBits = 16; break;
7356	case IEMMODE_32BIT: cAddrSizeBits = 32; break;
7357	case IEMMODE_64BIT: cAddrSizeBits = 64; break;
7358	IEM_NOT_REACHED_DEFAULT_CASE_RET();
7359	}
7360	rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_OUT, cbReg, cAddrSizeBits, 0 /* N/A - iEffSeg */,
7361	false /* fRep /, false / fStrIo */, cbInstr);
7362	if (rcStrict == VINF_SVM_VMEXIT)
7363	return VINF_SUCCESS;
7364	if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
7365	{
7366	Log(("iemCImpl_out: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, cbReg,
7367	VBOXSTRICTRC_VAL(rcStrict)));
7368	return rcStrict;
7369	}
7370	}
7371	#endif
7372
7373	/*
7374	* Perform the I/O.
7375	*/
7376	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
7377	uint32_t u32Value;
7378	switch (cbReg)
7379	{
7380	case 1: u32Value = pVCpu->cpum.GstCtx.al; break;
7381	case 2: u32Value = pVCpu->cpum.GstCtx.ax; break;
7382	case 4: u32Value = pVCpu->cpum.GstCtx.eax; break;
7383	default: AssertFailedReturn(VERR_IEM_IPE_4);
7384	}
7385	rcStrict = IOMIOPortWrite(pVM, pVCpu, u16Port, u32Value, cbReg);
7386	if (IOM_SUCCESS(rcStrict))
7387	{
7388	pVCpu->iem.s.cPotentialExits++;
7389	if (rcStrict != VINF_SUCCESS)
7390	iemSetPassUpStatus(pVCpu, rcStrict);
7391	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7392
7393	/*
7394	* Check for I/O breakpoints.
7395	*/
7396	/** @todo this should set a internal flag and be raised by
7397	* iemRegAddToRipAndFinishingClearingRF! */
7398	uint32_t const uDr7 = pVCpu->cpum.GstCtx.dr[7];
7399	if (RT_UNLIKELY( ( ( (uDr7 & X86_DR7_ENABLED_MASK)
7400	&& X86_DR7_ANY_RW_IO(uDr7)
7401	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE))
7402	\|\| DBGFBpIsHwIoArmed(pVM))
7403	&& rcStrict == VINF_SUCCESS))
7404	{
7405	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3 \| CPUMCTX_EXTRN_DR6);
7406	rcStrict = DBGFBpCheckIo(pVM, pVCpu, IEM_GET_CTX(pVCpu), u16Port, cbReg);
7407	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
7408	rcStrict = iemRaiseDebugException(pVCpu);
7409	}
7410	}
7411	return rcStrict;
7412	}
7413
7414
7415	/**
7416	* Implements 'OUT DX, eAX'.
7417	*
7418	* @param cbReg The register size.
7419	*/
7420	IEM_CIMPL_DEF_1(iemCImpl_out_DX_eAX, uint8_t, cbReg)
7421	{
7422	return IEM_CIMPL_CALL_3(iemCImpl_out, pVCpu->cpum.GstCtx.dx, false /* fImm */, cbReg);
7423	}
7424
7425
7426	/**
7427	* Implements 'CLI'.
7428	*/
7429	IEM_CIMPL_DEF_0(iemCImpl_cli)
7430	{
7431	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
7432	#ifdef LOG_ENABLED
7433	uint32_t const fEflOld = fEfl;
7434	#endif
7435
7436	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4);
7437	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
7438	{
7439	uint8_t const uIopl = X86_EFL_GET_IOPL(fEfl);
7440	if (!(fEfl & X86_EFL_VM))
7441	{
7442	if (pVCpu->iem.s.uCpl <= uIopl)
7443	fEfl &= ~X86_EFL_IF;
7444	else if ( pVCpu->iem.s.uCpl == 3
7445	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PVI) )
7446	fEfl &= ~X86_EFL_VIF;
7447	else
7448	return iemRaiseGeneralProtectionFault0(pVCpu);
7449	}
7450	/* V8086 */
7451	else if (uIopl == 3)
7452	fEfl &= ~X86_EFL_IF;
7453	else if ( uIopl < 3
7454	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) )
7455	fEfl &= ~X86_EFL_VIF;
7456	else
7457	return iemRaiseGeneralProtectionFault0(pVCpu);
7458	}
7459	/* real mode */
7460	else
7461	fEfl &= ~X86_EFL_IF;
7462
7463	/* Commit. */
7464	IEMMISC_SET_EFL(pVCpu, fEfl);
7465	VBOXSTRICTRC const rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7466	Log2(("CLI: %#x -> %#x\n", fEflOld, fEfl));
7467	return rcStrict;
7468	}
7469
7470
7471	/**
7472	* Implements 'STI'.
7473	*/
7474	IEM_CIMPL_DEF_0(iemCImpl_sti)
7475	{
7476	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
7477	uint32_t const fEflOld = fEfl;
7478
7479	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4);
7480	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
7481	{
7482	uint8_t const uIopl = X86_EFL_GET_IOPL(fEfl);
7483	if (!(fEfl & X86_EFL_VM))
7484	{
7485	if (pVCpu->iem.s.uCpl <= uIopl)
7486	fEfl \|= X86_EFL_IF;
7487	else if ( pVCpu->iem.s.uCpl == 3
7488	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PVI)
7489	&& !(fEfl & X86_EFL_VIP) )
7490	fEfl \|= X86_EFL_VIF;
7491	else
7492	return iemRaiseGeneralProtectionFault0(pVCpu);
7493	}
7494	/* V8086 */
7495	else if (uIopl == 3)
7496	fEfl \|= X86_EFL_IF;
7497	else if ( uIopl < 3
7498	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME)
7499	&& !(fEfl & X86_EFL_VIP) )
7500	fEfl \|= X86_EFL_VIF;
7501	else
7502	return iemRaiseGeneralProtectionFault0(pVCpu);
7503	}
7504	/* real mode */
7505	else
7506	fEfl \|= X86_EFL_IF;
7507
7508	/*
7509	* Commit.
7510	*
7511	* Note! Setting the shadow interrupt flag must be done after RIP updating.
7512	*/
7513	IEMMISC_SET_EFL(pVCpu, fEfl);
7514	VBOXSTRICTRC const rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7515	if (!(fEflOld & X86_EFL_IF) && (fEfl & X86_EFL_IF))
7516	{
7517	/** @todo only set it the shadow flag if it was clear before? */
7518	CPUMSetInInterruptShadowSti(&pVCpu->cpum.GstCtx);
7519	}
7520	Log2(("STI: %#x -> %#x\n", fEflOld, fEfl));
7521	return rcStrict;
7522	}
7523
7524
7525	/**
7526	* Implements 'HLT'.
7527	*/
7528	IEM_CIMPL_DEF_0(iemCImpl_hlt)
7529	{
7530	if (pVCpu->iem.s.uCpl != 0)
7531	return iemRaiseGeneralProtectionFault0(pVCpu);
7532
7533	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7534	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_HLT_EXIT))
7535	{
7536	Log2(("hlt: Guest intercept -> VM-exit\n"));
7537	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_HLT, cbInstr);
7538	}
7539
7540	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_HLT))
7541	{
7542	Log2(("hlt: Guest intercept -> #VMEXIT\n"));
7543	IEM_SVM_UPDATE_NRIP(pVCpu);
7544	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_HLT, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7545	}
7546
7547	/** @todo finish: This ASSUMES that iemRegAddToRipAndFinishingClearingRF won't
7548	* be returning any status codes relating to non-guest events being raised, as
7549	* we'll mess up the guest HALT otherwise. */
7550	VBOXSTRICTRC rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7551	if (rcStrict == VINF_SUCCESS)
7552	rcStrict = VINF_EM_HALT;
7553	return rcStrict;
7554	}
7555
7556
7557	/**
7558	* Implements 'MONITOR'.
7559	*/
7560	IEM_CIMPL_DEF_1(iemCImpl_monitor, uint8_t, iEffSeg)
7561	{
7562	/*
7563	* Permission checks.
7564	*/
7565	if (pVCpu->iem.s.uCpl != 0)
7566	{
7567	Log2(("monitor: CPL != 0\n"));
7568	return iemRaiseUndefinedOpcode(pVCpu); /** @todo MSR[0xC0010015].MonMwaitUserEn if we care. */
7569	}
7570	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMonitorMWait)
7571	{
7572	Log2(("monitor: Not in CPUID\n"));
7573	return iemRaiseUndefinedOpcode(pVCpu);
7574	}
7575
7576	/*
7577	* Check VMX guest-intercept.
7578	* This should be considered a fault-like VM-exit.
7579	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7580	*/
7581	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7582	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_MONITOR_EXIT))
7583	{
7584	Log2(("monitor: Guest intercept -> #VMEXIT\n"));
7585	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_MONITOR, cbInstr);
7586	}
7587
7588	/*
7589	* Gather the operands and validate them.
7590	*/
7591	RTGCPTR GCPtrMem = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
7592	uint32_t uEcx = pVCpu->cpum.GstCtx.ecx;
7593	uint32_t uEdx = pVCpu->cpum.GstCtx.edx;
7594	/** @todo Test whether EAX or ECX is processed first, i.e. do we get \#PF or
7595	* \#GP first. */
7596	if (uEcx != 0)
7597	{
7598	Log2(("monitor rax=%RX64, ecx=%RX32, edx=%RX32; ECX != 0 -> #GP(0)\n", GCPtrMem, uEcx, uEdx)); NOREF(uEdx);
7599	return iemRaiseGeneralProtectionFault0(pVCpu);
7600	}
7601
7602	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA, iEffSeg, 1, &GCPtrMem);
7603	if (rcStrict != VINF_SUCCESS)
7604	return rcStrict;
7605
7606	RTGCPHYS GCPhysMem;
7607	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA, &GCPhysMem);
7608	if (rcStrict != VINF_SUCCESS)
7609	return rcStrict;
7610
7611	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7612	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7613	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
7614	{
7615	/*
7616	* MONITOR does not access the memory, just monitors the address. However,
7617	* if the address falls in the APIC-access page, the address monitored must
7618	* instead be the corresponding address in the virtual-APIC page.
7619	*
7620	* See Intel spec. 29.4.4 "Instruction-Specific Considerations".
7621	*/
7622	rcStrict = iemVmxVirtApicAccessUnused(pVCpu, &GCPhysMem, 1, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA);
7623	if ( rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE
7624	&& rcStrict != VINF_VMX_MODIFIES_BEHAVIOR)
7625	return rcStrict;
7626	}
7627	#endif
7628
7629	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MONITOR))
7630	{
7631	Log2(("monitor: Guest intercept -> #VMEXIT\n"));
7632	IEM_SVM_UPDATE_NRIP(pVCpu);
7633	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MONITOR, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7634	}
7635
7636	/*
7637	* Call EM to prepare the monitor/wait.
7638	*/
7639	rcStrict = EMMonitorWaitPrepare(pVCpu, pVCpu->cpum.GstCtx.rax, pVCpu->cpum.GstCtx.rcx, pVCpu->cpum.GstCtx.rdx, GCPhysMem);
7640	Assert(rcStrict == VINF_SUCCESS);
7641	if (rcStrict == VINF_SUCCESS)
7642	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7643	return rcStrict;
7644	}
7645
7646
7647	/**
7648	* Implements 'MWAIT'.
7649	*/
7650	IEM_CIMPL_DEF_0(iemCImpl_mwait)
7651	{
7652	/*
7653	* Permission checks.
7654	*/
7655	if (pVCpu->iem.s.uCpl != 0)
7656	{
7657	Log2(("mwait: CPL != 0\n"));
7658	/** @todo MSR[0xC0010015].MonMwaitUserEn if we care. (Remember to check
7659	* EFLAGS.VM then.) */
7660	return iemRaiseUndefinedOpcode(pVCpu);
7661	}
7662	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMonitorMWait)
7663	{
7664	Log2(("mwait: Not in CPUID\n"));
7665	return iemRaiseUndefinedOpcode(pVCpu);
7666	}
7667
7668	/* Check VMX nested-guest intercept. */
7669	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7670	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_MWAIT_EXIT))
7671	IEM_VMX_VMEXIT_MWAIT_RET(pVCpu, EMMonitorIsArmed(pVCpu), cbInstr);
7672
7673	/*
7674	* Gather the operands and validate them.
7675	*/
7676	uint32_t const uEax = pVCpu->cpum.GstCtx.eax;
7677	uint32_t const uEcx = pVCpu->cpum.GstCtx.ecx;
7678	if (uEcx != 0)
7679	{
7680	/* Only supported extension is break on IRQ when IF=0. */
7681	if (uEcx > 1)
7682	{
7683	Log2(("mwait eax=%RX32, ecx=%RX32; ECX > 1 -> #GP(0)\n", uEax, uEcx));
7684	return iemRaiseGeneralProtectionFault0(pVCpu);
7685	}
7686	uint32_t fMWaitFeatures = 0;
7687	uint32_t uIgnore = 0;
7688	CPUMGetGuestCpuId(pVCpu, 5, 0, -1 /f64BitMode/, &uIgnore, &uIgnore, &fMWaitFeatures, &uIgnore);
7689	if ( (fMWaitFeatures & (X86_CPUID_MWAIT_ECX_EXT \| X86_CPUID_MWAIT_ECX_BREAKIRQIF0))
7690	!= (X86_CPUID_MWAIT_ECX_EXT \| X86_CPUID_MWAIT_ECX_BREAKIRQIF0))
7691	{
7692	Log2(("mwait eax=%RX32, ecx=%RX32; break-on-IRQ-IF=0 extension not enabled -> #GP(0)\n", uEax, uEcx));
7693	return iemRaiseGeneralProtectionFault0(pVCpu);
7694	}
7695
7696	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7697	/*
7698	* If the interrupt-window exiting control is set or a virtual-interrupt is pending
7699	* for delivery; and interrupts are disabled the processor does not enter its
7700	* mwait state but rather passes control to the next instruction.
7701	*
7702	* See Intel spec. 25.3 "Changes to Instruction Behavior In VMX Non-root Operation".
7703	*/
7704	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7705	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1IF)
7706	{
7707	if ( IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INT_WINDOW_EXIT)
7708	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
7709	/** @todo finish: check up this out after we move int window stuff out of the
7710	* run loop and into the instruction finishing logic here. */
7711	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7712	}
7713	#endif
7714	}
7715
7716	/*
7717	* Check SVM nested-guest mwait intercepts.
7718	*/
7719	if ( IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MWAIT_ARMED)
7720	&& EMMonitorIsArmed(pVCpu))
7721	{
7722	Log2(("mwait: Guest intercept (monitor hardware armed) -> #VMEXIT\n"));
7723	IEM_SVM_UPDATE_NRIP(pVCpu);
7724	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MWAIT_ARMED, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7725	}
7726	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MWAIT))
7727	{
7728	Log2(("mwait: Guest intercept -> #VMEXIT\n"));
7729	IEM_SVM_UPDATE_NRIP(pVCpu);
7730	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MWAIT, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7731	}
7732
7733	/*
7734	* Call EM to prepare the monitor/wait.
7735	*
7736	* This will return VINF_EM_HALT. If there the trap flag is set, we may
7737	* override it when executing iemRegAddToRipAndFinishingClearingRF ASSUMING
7738	* that will only return guest related events.
7739	*/
7740	VBOXSTRICTRC rcStrict = EMMonitorWaitPerform(pVCpu, uEax, uEcx);
7741
7742	/** @todo finish: This needs more thinking as we should suppress internal
7743	* debugger events here, or we'll bugger up the guest state even more than we
7744	* alread do around VINF_EM_HALT. */
7745	VBOXSTRICTRC rcStrict2 = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7746	if (rcStrict2 != VINF_SUCCESS)
7747	{
7748	Log2(("mwait: %Rrc (perform) -> %Rrc (finish)!\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2) ));
7749	rcStrict = rcStrict2;
7750	}
7751
7752	return rcStrict;
7753	}
7754
7755
7756	/**
7757	* Implements 'SWAPGS'.
7758	*/
7759	IEM_CIMPL_DEF_0(iemCImpl_swapgs)
7760	{
7761	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT); /* Caller checks this. */
7762
7763	/*
7764	* Permission checks.
7765	*/
7766	if (pVCpu->iem.s.uCpl != 0)
7767	{
7768	Log2(("swapgs: CPL != 0\n"));
7769	return iemRaiseUndefinedOpcode(pVCpu);
7770	}
7771
7772	/*
7773	* Do the job.
7774	*/
7775	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_GS);
7776	uint64_t uOtherGsBase = pVCpu->cpum.GstCtx.msrKERNELGSBASE;
7777	pVCpu->cpum.GstCtx.msrKERNELGSBASE = pVCpu->cpum.GstCtx.gs.u64Base;
7778	pVCpu->cpum.GstCtx.gs.u64Base = uOtherGsBase;
7779
7780	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7781	}
7782
7783
7784	#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
7785	/**
7786	* Handles a CPUID call.
7787	*/
7788	static VBOXSTRICTRC iemCpuIdVBoxCall(PVMCPUCC pVCpu, uint32_t iFunction,
7789	uint32_t pEax, uint32_t pEbx, uint32_t pEcx, uint32_t pEdx)
7790	{
7791	switch (iFunction)
7792	{
7793	case VBOX_CPUID_FN_ID:
7794	LogFlow(("iemCpuIdVBoxCall: VBOX_CPUID_FN_ID\n"));
7795	*pEax = VBOX_CPUID_RESP_ID_EAX;
7796	*pEbx = VBOX_CPUID_RESP_ID_EBX;
7797	*pEcx = VBOX_CPUID_RESP_ID_ECX;
7798	*pEdx = VBOX_CPUID_RESP_ID_EDX;
7799	break;
7800
7801	case VBOX_CPUID_FN_LOG:
7802	{
7803	CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX \| CPUMCTX_EXTRN_RSI
7804	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7805
7806	/* Validate input. */
7807	uint32_t cchToLog = *pEdx;
7808	if (cchToLog <= _2M)
7809	{
7810	uint32_t const uLogPicker = *pEbx;
7811	if (uLogPicker <= 1)
7812	{
7813	/* Resolve the logger. */
7814	PRTLOGGER const pLogger = !uLogPicker
7815	? RTLogDefaultInstanceEx(UINT32_MAX) : RTLogRelGetDefaultInstanceEx(UINT32_MAX);
7816	if (pLogger)
7817	{
7818	/* Copy over the data: */
7819	RTGCPTR GCPtrSrc = pVCpu->cpum.GstCtx.rsi;
7820	while (cchToLog > 0)
7821	{
7822	uint32_t cbToMap = GUEST_PAGE_SIZE - (GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
7823	if (cbToMap > cchToLog)
7824	cbToMap = cchToLog;
7825	/** @todo Extend iemMemMap to allowing page size accessing and avoid 7
7826	* unnecessary calls & iterations per pages. */
7827	if (cbToMap > 512)
7828	cbToMap = 512;
7829	void *pvSrc = NULL;
7830	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvSrc, cbToMap, UINT8_MAX, GCPtrSrc, IEM_ACCESS_DATA_R, 0);
7831	if (rcStrict == VINF_SUCCESS)
7832	{
7833	RTLogBulkNestedWrite(pLogger, (const char *)pvSrc, cbToMap, "Gst:");
7834	rcStrict = iemMemCommitAndUnmap(pVCpu, pvSrc, IEM_ACCESS_DATA_R);
7835	AssertRCSuccessReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7836	}
7837	else
7838	{
7839	Log(("iemCpuIdVBoxCall: %Rrc at %RGp LB %#x\n", VBOXSTRICTRC_VAL(rcStrict), GCPtrSrc, cbToMap));
7840	return rcStrict;
7841	}
7842
7843	/* Advance. */
7844	pVCpu->cpum.GstCtx.rsi = GCPtrSrc += cbToMap;
7845	*pEdx = cchToLog -= cbToMap;
7846	}
7847	*pEax = VINF_SUCCESS;
7848	}
7849	else
7850	*pEax = (uint32_t)VERR_NOT_FOUND;
7851	}
7852	else
7853	*pEax = (uint32_t)VERR_NOT_FOUND;
7854	}
7855	else
7856	*pEax = (uint32_t)VERR_TOO_MUCH_DATA;
7857	*pEdx = VBOX_CPUID_RESP_GEN_EDX;
7858	*pEcx = VBOX_CPUID_RESP_GEN_ECX;
7859	*pEbx = VBOX_CPUID_RESP_GEN_EBX;
7860	break;
7861	}
7862
7863	default:
7864	LogFlow(("iemCpuIdVBoxCall: Invalid function %#x (%#x, %#x)\n", iFunction, pEbx, pEdx));
7865	*pEax = (uint32_t)VERR_INVALID_FUNCTION;
7866	*pEbx = (uint32_t)VERR_INVALID_FUNCTION;
7867	*pEcx = (uint32_t)VERR_INVALID_FUNCTION;
7868	*pEdx = (uint32_t)VERR_INVALID_FUNCTION;
7869	break;
7870	}
7871	return VINF_SUCCESS;
7872	}
7873	#endif /* VBOX_WITHOUT_CPUID_HOST_CALL */
7874
7875	/**
7876	* Implements 'CPUID'.
7877	*/
7878	IEM_CIMPL_DEF_0(iemCImpl_cpuid)
7879	{
7880	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7881	{
7882	Log2(("cpuid: Guest intercept -> VM-exit\n"));
7883	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_CPUID, cbInstr);
7884	}
7885
7886	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CPUID))
7887	{
7888	Log2(("cpuid: Guest intercept -> #VMEXIT\n"));
7889	IEM_SVM_UPDATE_NRIP(pVCpu);
7890	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_CPUID, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7891	}
7892
7893
7894	uint32_t const uEax = pVCpu->cpum.GstCtx.eax;
7895	uint32_t const uEcx = pVCpu->cpum.GstCtx.ecx;
7896
7897	#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
7898	/*
7899	* CPUID host call backdoor.
7900	*/
7901	if ( uEax == VBOX_CPUID_REQ_EAX_FIXED
7902	&& (uEcx & VBOX_CPUID_REQ_ECX_FIXED_MASK) == VBOX_CPUID_REQ_ECX_FIXED
7903	&& pVCpu->CTX_SUFF(pVM)->iem.s.fCpuIdHostCall)
7904	{
7905	VBOXSTRICTRC rcStrict = iemCpuIdVBoxCall(pVCpu, uEcx & VBOX_CPUID_REQ_ECX_FN_MASK,
7906	&pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx,
7907	&pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
7908	if (rcStrict != VINF_SUCCESS)
7909	return rcStrict;
7910	}
7911	/*
7912	* Regular CPUID.
7913	*/
7914	else
7915	#endif
7916	CPUMGetGuestCpuId(pVCpu, uEax, uEcx, pVCpu->cpum.GstCtx.cs.Attr.n.u1Long,
7917	&pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
7918
7919	pVCpu->cpum.GstCtx.rax &= UINT32_C(0xffffffff);
7920	pVCpu->cpum.GstCtx.rbx &= UINT32_C(0xffffffff);
7921	pVCpu->cpum.GstCtx.rcx &= UINT32_C(0xffffffff);
7922	pVCpu->cpum.GstCtx.rdx &= UINT32_C(0xffffffff);
7923	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
7924
7925	pVCpu->iem.s.cPotentialExits++;
7926	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7927	}
7928
7929
7930	/**
7931	* Implements 'AAD'.
7932	*
7933	* @param bImm The immediate operand.
7934	*/
7935	IEM_CIMPL_DEF_1(iemCImpl_aad, uint8_t, bImm)
7936	{
7937	uint16_t const ax = pVCpu->cpum.GstCtx.ax;
7938	uint8_t const al = (uint8_t)ax + (uint8_t)(ax >> 8) * bImm;
7939	pVCpu->cpum.GstCtx.ax = al;
7940	iemHlpUpdateArithEFlagsU8(pVCpu, al,
7941	X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF,
7942	X86_EFL_OF \| X86_EFL_AF \| X86_EFL_CF);
7943
7944	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7945	}
7946
7947
7948	/**
7949	* Implements 'AAM'.
7950	*
7951	* @param bImm The immediate operand. Cannot be 0.
7952	*/
7953	IEM_CIMPL_DEF_1(iemCImpl_aam, uint8_t, bImm)
7954	{
7955	Assert(bImm != 0); /* #DE on 0 is handled in the decoder. */
7956
7957	uint16_t const ax = pVCpu->cpum.GstCtx.ax;
7958	uint8_t const al = (uint8_t)ax % bImm;
7959	uint8_t const ah = (uint8_t)ax / bImm;
7960	pVCpu->cpum.GstCtx.ax = (ah << 8) + al;
7961	iemHlpUpdateArithEFlagsU8(pVCpu, al,
7962	X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF,
7963	X86_EFL_OF \| X86_EFL_AF \| X86_EFL_CF);
7964
7965	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7966	}
7967
7968
7969	/**
7970	* Implements 'DAA'.
7971	*/
7972	IEM_CIMPL_DEF_0(iemCImpl_daa)
7973	{
7974	uint8_t const al = pVCpu->cpum.GstCtx.al;
7975	bool const fCarry = pVCpu->cpum.GstCtx.eflags.Bits.u1CF;
7976
7977	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
7978	\|\| (al & 0xf) >= 10)
7979	{
7980	pVCpu->cpum.GstCtx.al = al + 6;
7981	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
7982	}
7983	else
7984	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
7985
7986	if (al >= 0x9a \|\| fCarry)
7987	{
7988	pVCpu->cpum.GstCtx.al += 0x60;
7989	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
7990	}
7991	else
7992	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
7993
7994	iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
7995	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7996	}
7997
7998
7999	/**
8000	* Implements 'DAS'.
8001	*/
8002	IEM_CIMPL_DEF_0(iemCImpl_das)
8003	{
8004	uint8_t const uInputAL = pVCpu->cpum.GstCtx.al;
8005	bool const fCarry = pVCpu->cpum.GstCtx.eflags.Bits.u1CF;
8006
8007	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8008	\|\| (uInputAL & 0xf) >= 10)
8009	{
8010	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8011	if (uInputAL < 6)
8012	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8013	pVCpu->cpum.GstCtx.al = uInputAL - 6;
8014	}
8015	else
8016	{
8017	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8018	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8019	}
8020
8021	if (uInputAL >= 0x9a \|\| fCarry)
8022	{
8023	pVCpu->cpum.GstCtx.al -= 0x60;
8024	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8025	}
8026
8027	iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8028	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8029	}
8030
8031
8032	/**
8033	* Implements 'AAA'.
8034	*/
8035	IEM_CIMPL_DEF_0(iemCImpl_aaa)
8036	{
8037	if (IEM_IS_GUEST_CPU_AMD(pVCpu))
8038	{
8039	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8040	\|\| (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
8041	{
8042	iemAImpl_add_u16(&pVCpu->cpum.GstCtx.ax, 0x106, &pVCpu->cpum.GstCtx.eflags.uBoth);
8043	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8044	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8045	}
8046	else
8047	{
8048	iemHlpUpdateArithEFlagsU16(pVCpu, pVCpu->cpum.GstCtx.ax, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8049	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8050	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8051	}
8052	pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
8053	}
8054	else
8055	{
8056	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8057	\|\| (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
8058	{
8059	pVCpu->cpum.GstCtx.ax += UINT16_C(0x106);
8060	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8061	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8062	}
8063	else
8064	{
8065	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8066	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8067	}
8068	pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
8069	iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8070	}
8071
8072	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8073	}
8074
8075
8076	/**
8077	* Implements 'AAS'.
8078	*/
8079	IEM_CIMPL_DEF_0(iemCImpl_aas)
8080	{
8081	if (IEM_IS_GUEST_CPU_AMD(pVCpu))
8082	{
8083	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8084	\|\| (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
8085	{
8086	iemAImpl_sub_u16(&pVCpu->cpum.GstCtx.ax, 0x106, &pVCpu->cpum.GstCtx.eflags.uBoth);
8087	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8088	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8089	}
8090	else
8091	{
8092	iemHlpUpdateArithEFlagsU16(pVCpu, pVCpu->cpum.GstCtx.ax, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8093	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8094	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8095	}
8096	pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
8097	}
8098	else
8099	{
8100	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8101	\|\| (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
8102	{
8103	pVCpu->cpum.GstCtx.ax -= UINT16_C(0x106);
8104	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8105	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8106	}
8107	else
8108	{
8109	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8110	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8111	}
8112	pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
8113	iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8114	}
8115
8116	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8117	}
8118
8119
8120	/**
8121	* Implements the 16-bit version of 'BOUND'.
8122	*
8123	* @note We have separate 16-bit and 32-bit variants of this function due to
8124	* the decoder using unsigned parameters, whereas we want signed one to
8125	* do the job. This is significant for a recompiler.
8126	*/
8127	IEM_CIMPL_DEF_3(iemCImpl_bound_16, int16_t, idxArray, int16_t, idxLowerBound, int16_t, idxUpperBound)
8128	{
8129	/*
8130	* Check if the index is inside the bounds, otherwise raise #BR.
8131	*/
8132	if ( idxArray >= idxLowerBound
8133	&& idxArray <= idxUpperBound)
8134	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8135	return iemRaiseBoundRangeExceeded(pVCpu);
8136	}
8137
8138
8139	/**
8140	* Implements the 32-bit version of 'BOUND'.
8141	*/
8142	IEM_CIMPL_DEF_3(iemCImpl_bound_32, int32_t, idxArray, int32_t, idxLowerBound, int32_t, idxUpperBound)
8143	{
8144	/*
8145	* Check if the index is inside the bounds, otherwise raise #BR.
8146	*/
8147	if ( idxArray >= idxLowerBound
8148	&& idxArray <= idxUpperBound)
8149	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8150	return iemRaiseBoundRangeExceeded(pVCpu);
8151	}
8152
8153
8154
8155	/*
8156	* Instantiate the various string operation combinations.
8157	*/
8158	#define OP_SIZE 8
8159	#define ADDR_SIZE 16
8160	#include "IEMAllCImplStrInstr.cpp.h"
8161	#define OP_SIZE 8
8162	#define ADDR_SIZE 32
8163	#include "IEMAllCImplStrInstr.cpp.h"
8164	#define OP_SIZE 8
8165	#define ADDR_SIZE 64
8166	#include "IEMAllCImplStrInstr.cpp.h"
8167
8168	#define OP_SIZE 16
8169	#define ADDR_SIZE 16
8170	#include "IEMAllCImplStrInstr.cpp.h"
8171	#define OP_SIZE 16
8172	#define ADDR_SIZE 32
8173	#include "IEMAllCImplStrInstr.cpp.h"
8174	#define OP_SIZE 16
8175	#define ADDR_SIZE 64
8176	#include "IEMAllCImplStrInstr.cpp.h"
8177
8178	#define OP_SIZE 32
8179	#define ADDR_SIZE 16
8180	#include "IEMAllCImplStrInstr.cpp.h"
8181	#define OP_SIZE 32
8182	#define ADDR_SIZE 32
8183	#include "IEMAllCImplStrInstr.cpp.h"
8184	#define OP_SIZE 32
8185	#define ADDR_SIZE 64
8186	#include "IEMAllCImplStrInstr.cpp.h"
8187
8188	#define OP_SIZE 64
8189	#define ADDR_SIZE 32
8190	#include "IEMAllCImplStrInstr.cpp.h"
8191	#define OP_SIZE 64
8192	#define ADDR_SIZE 64
8193	#include "IEMAllCImplStrInstr.cpp.h"
8194
8195
8196	/**
8197	* Implements 'XGETBV'.
8198	*/
8199	IEM_CIMPL_DEF_0(iemCImpl_xgetbv)
8200	{
8201	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
8202	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)
8203	{
8204	uint32_t uEcx = pVCpu->cpum.GstCtx.ecx;
8205	switch (uEcx)
8206	{
8207	case 0:
8208	break;
8209
8210	case 1: /** @todo Implement XCR1 support. */
8211	default:
8212	Log(("xgetbv ecx=%RX32 -> #GP(0)\n", uEcx));
8213	return iemRaiseGeneralProtectionFault0(pVCpu);
8214
8215	}
8216	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_XCRx);
8217	pVCpu->cpum.GstCtx.rax = RT_LO_U32(pVCpu->cpum.GstCtx.aXcr[uEcx]);
8218	pVCpu->cpum.GstCtx.rdx = RT_HI_U32(pVCpu->cpum.GstCtx.aXcr[uEcx]);
8219
8220	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8221	}
8222	Log(("xgetbv CR4.OSXSAVE=0 -> UD\n"));
8223	return iemRaiseUndefinedOpcode(pVCpu);
8224	}
8225
8226
8227	/**
8228	* Implements 'XSETBV'.
8229	*/
8230	IEM_CIMPL_DEF_0(iemCImpl_xsetbv)
8231	{
8232	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)
8233	{
8234	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_XSETBV))
8235	{
8236	Log2(("xsetbv: Guest intercept -> #VMEXIT\n"));
8237	IEM_SVM_UPDATE_NRIP(pVCpu);
8238	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XSETBV, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
8239	}
8240
8241	if (pVCpu->iem.s.uCpl == 0)
8242	{
8243	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_XCRx);
8244
8245	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
8246	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_XSETBV, cbInstr);
8247
8248	uint32_t uEcx = pVCpu->cpum.GstCtx.ecx;
8249	uint64_t uNewValue = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx);
8250	switch (uEcx)
8251	{
8252	case 0:
8253	{
8254	int rc = CPUMSetGuestXcr0(pVCpu, uNewValue);
8255	if (rc == VINF_SUCCESS)
8256	break;
8257	Assert(rc == VERR_CPUM_RAISE_GP_0);
8258	Log(("xsetbv ecx=%RX32 (newvalue=%RX64) -> #GP(0)\n", uEcx, uNewValue));
8259	return iemRaiseGeneralProtectionFault0(pVCpu);
8260	}
8261
8262	case 1: /** @todo Implement XCR1 support. */
8263	default:
8264	Log(("xsetbv ecx=%RX32 (newvalue=%RX64) -> #GP(0)\n", uEcx, uNewValue));
8265	return iemRaiseGeneralProtectionFault0(pVCpu);
8266
8267	}
8268
8269	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8270	}
8271
8272	Log(("xsetbv cpl=%u -> GP(0)\n", pVCpu->iem.s.uCpl));
8273	return iemRaiseGeneralProtectionFault0(pVCpu);
8274	}
8275	Log(("xsetbv CR4.OSXSAVE=0 -> UD\n"));
8276	return iemRaiseUndefinedOpcode(pVCpu);
8277	}
8278
8279	#ifndef RT_ARCH_ARM64
8280	# ifdef IN_RING3
8281
8282	/** Argument package for iemCImpl_cmpxchg16b_fallback_rendezvous_callback. */
8283	struct IEMCIMPLCX16ARGS
8284	{
8285	PRTUINT128U pu128Dst;
8286	PRTUINT128U pu128RaxRdx;
8287	PRTUINT128U pu128RbxRcx;
8288	uint32_t *pEFlags;
8289	# ifdef VBOX_STRICT
8290	uint32_t cCalls;
8291	# endif
8292	};
8293
8294	/**
8295	* @callback_method_impl{FNVMMEMTRENDEZVOUS,
8296	* Worker for iemCImpl_cmpxchg16b_fallback_rendezvous}
8297	*/
8298	static DECLCALLBACK(VBOXSTRICTRC) iemCImpl_cmpxchg16b_fallback_rendezvous_callback(PVM pVM, PVMCPUCC pVCpu, void *pvUser)
8299	{
8300	RT_NOREF(pVM, pVCpu);
8301	struct IEMCIMPLCX16ARGS pArgs = (struct IEMCIMPLCX16ARGS )pvUser;
8302	# ifdef VBOX_STRICT
8303	Assert(pArgs->cCalls == 0);
8304	pArgs->cCalls++;
8305	# endif
8306
8307	iemAImpl_cmpxchg16b_fallback(pArgs->pu128Dst, pArgs->pu128RaxRdx, pArgs->pu128RbxRcx, pArgs->pEFlags);
8308	return VINF_SUCCESS;
8309	}
8310
8311	# endif /* IN_RING3 */
8312
8313	/**
8314	* Implements 'CMPXCHG16B' fallback using rendezvous.
8315	*/
8316	IEM_CIMPL_DEF_4(iemCImpl_cmpxchg16b_fallback_rendezvous, PRTUINT128U, pu128Dst, PRTUINT128U, pu128RaxRdx,
8317	PRTUINT128U, pu128RbxRcx, uint32_t *, pEFlags)
8318	{
8319	# ifdef IN_RING3
8320	struct IEMCIMPLCX16ARGS Args;
8321	Args.pu128Dst = pu128Dst;
8322	Args.pu128RaxRdx = pu128RaxRdx;
8323	Args.pu128RbxRcx = pu128RbxRcx;
8324	Args.pEFlags = pEFlags;
8325	# ifdef VBOX_STRICT
8326	Args.cCalls = 0;
8327	# endif
8328	VBOXSTRICTRC rcStrict = VMMR3EmtRendezvous(pVCpu->CTX_SUFF(pVM), VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
8329	iemCImpl_cmpxchg16b_fallback_rendezvous_callback, &Args);
8330	Assert(Args.cCalls == 1);
8331	if (rcStrict == VINF_SUCCESS)
8332	{
8333	/* Duplicated tail code. */
8334	rcStrict = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_RW);
8335	if (rcStrict == VINF_SUCCESS)
8336	{
8337	pVCpu->cpum.GstCtx.eflags.u = pEFlags; / IEM_MC_COMMIT_EFLAGS */
8338	if (!(*pEFlags & X86_EFL_ZF))
8339	{
8340	pVCpu->cpum.GstCtx.rax = pu128RaxRdx->s.Lo;
8341	pVCpu->cpum.GstCtx.rdx = pu128RaxRdx->s.Hi;
8342	}
8343	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8344	}
8345	}
8346	return rcStrict;
8347	# else
8348	RT_NOREF(pVCpu, cbInstr, pu128Dst, pu128RaxRdx, pu128RbxRcx, pEFlags);
8349	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; /* This should get us to ring-3 for now. Should perhaps be replaced later. */
8350	# endif
8351	}
8352
8353	#endif /* RT_ARCH_ARM64 */
8354
8355	/**
8356	* Implements 'CLFLUSH' and 'CLFLUSHOPT'.
8357	*
8358	* This is implemented in C because it triggers a load like behaviour without
8359	* actually reading anything. Since that's not so common, it's implemented
8360	* here.
8361	*
8362	* @param iEffSeg The effective segment.
8363	* @param GCPtrEff The address of the image.
8364	*/
8365	IEM_CIMPL_DEF_2(iemCImpl_clflush_clflushopt, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
8366	{
8367	/*
8368	* Pretend to do a load w/o reading (see also iemCImpl_monitor and iemMemMap).
8369	*/
8370	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA, iEffSeg, 1, &GCPtrEff);
8371	if (rcStrict == VINF_SUCCESS)
8372	{
8373	RTGCPHYS GCPhysMem;
8374	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrEff, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA, &GCPhysMem);
8375	if (rcStrict == VINF_SUCCESS)
8376	{
8377	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8378	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
8379	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
8380	{
8381	/*
8382	* CLFLUSH/CLFLUSHOPT does not access the memory, but flushes the cache-line
8383	* that contains the address. However, if the address falls in the APIC-access
8384	* page, the address flushed must instead be the corresponding address in the
8385	* virtual-APIC page.
8386	*
8387	* See Intel spec. 29.4.4 "Instruction-Specific Considerations".
8388	*/
8389	rcStrict = iemVmxVirtApicAccessUnused(pVCpu, &GCPhysMem, 1, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA);
8390	if ( rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE
8391	&& rcStrict != VINF_VMX_MODIFIES_BEHAVIOR)
8392	return rcStrict;
8393	}
8394	#endif
8395	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8396	}
8397	}
8398
8399	return rcStrict;
8400	}
8401
8402
8403	/**
8404	* Implements 'FINIT' and 'FNINIT'.
8405	*
8406	* @param fCheckXcpts Whether to check for umasked pending exceptions or
8407	* not.
8408	*/
8409	IEM_CIMPL_DEF_1(iemCImpl_finit, bool, fCheckXcpts)
8410	{
8411	/*
8412	* Exceptions.
8413	*/
8414	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
8415	if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS))
8416	return iemRaiseDeviceNotAvailable(pVCpu);
8417
8418	iemFpuActualizeStateForChange(pVCpu);
8419	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_X87);
8420
8421	/* FINIT: Raise #MF on pending exception(s): */
8422	if (fCheckXcpts && (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))
8423	return iemRaiseMathFault(pVCpu);
8424
8425	/*
8426	* Reset the state.
8427	*/
8428	PX86XSAVEAREA pXState = &pVCpu->cpum.GstCtx.XState;
8429
8430	/* Rotate the stack to account for changed TOS. */
8431	iemFpuRotateStackSetTop(&pXState->x87, 0);
8432
8433	pXState->x87.FCW = 0x37f;
8434	pXState->x87.FSW = 0;
8435	pXState->x87.FTW = 0x00; /* 0 - empty. */
8436	/** @todo Intel says the instruction and data pointers are not cleared on
8437	* 387, presume that 8087 and 287 doesn't do so either. */
8438	/** @todo test this stuff. */
8439	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
8440	{
8441	pXState->x87.FPUDP = 0;
8442	pXState->x87.DS = 0; //??
8443	pXState->x87.Rsrvd2 = 0;
8444	pXState->x87.FPUIP = 0;
8445	pXState->x87.CS = 0; //??
8446	pXState->x87.Rsrvd1 = 0;
8447	}
8448	pXState->x87.FOP = 0;
8449
8450	iemHlpUsedFpu(pVCpu);
8451	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8452	}
8453
8454
8455	/**
8456	* Implements 'FXSAVE'.
8457	*
8458	* @param iEffSeg The effective segment.
8459	* @param GCPtrEff The address of the image.
8460	* @param enmEffOpSize The operand size (only REX.W really matters).
8461	*/
8462	IEM_CIMPL_DEF_3(iemCImpl_fxsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
8463	{
8464	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX);
8465
8466	/*
8467	* Raise exceptions.
8468	*/
8469	if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_TS \| X86_CR0_EM))
8470	return iemRaiseDeviceNotAvailable(pVCpu);
8471
8472	/*
8473	* Access the memory.
8474	*/
8475	void *pvMem512;
8476	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE,
8477	15 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_GP_OR_AC);
8478	if (rcStrict != VINF_SUCCESS)
8479	return rcStrict;
8480	PX86FXSTATE pDst = (PX86FXSTATE)pvMem512;
8481	PCX86FXSTATE pSrc = &pVCpu->cpum.GstCtx.XState.x87;
8482
8483	/*
8484	* Store the registers.
8485	*/
8486	/** @todo CPU/VM detection possible! If CR4.OSFXSR=0 MXCSR it's
8487	* implementation specific whether MXCSR and XMM0-XMM7 are saved. */
8488
8489	/* common for all formats */
8490	pDst->FCW = pSrc->FCW;
8491	pDst->FSW = pSrc->FSW;
8492	pDst->FTW = pSrc->FTW & UINT16_C(0xff);
8493	pDst->FOP = pSrc->FOP;
8494	pDst->MXCSR = pSrc->MXCSR;
8495	pDst->MXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
8496	for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
8497	{
8498	/** @todo Testcase: What actually happens to the 6 reserved bytes? I'm clearing
8499	* them for now... */
8500	pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
8501	pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
8502	pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
8503	pDst->aRegs[i].au32[3] = 0;
8504	}
8505
8506	/* FPU IP, CS, DP and DS. */
8507	pDst->FPUIP = pSrc->FPUIP;
8508	pDst->CS = pSrc->CS;
8509	pDst->FPUDP = pSrc->FPUDP;
8510	pDst->DS = pSrc->DS;
8511	if (enmEffOpSize == IEMMODE_64BIT)
8512	{
8513	/* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
8514	pDst->Rsrvd1 = pSrc->Rsrvd1;
8515	pDst->Rsrvd2 = pSrc->Rsrvd2;
8516	}
8517	else
8518	{
8519	pDst->Rsrvd1 = 0;
8520	pDst->Rsrvd2 = 0;
8521	}
8522
8523	/* XMM registers. */
8524	if ( !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_FFXSR)
8525	\|\| pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
8526	\|\| pVCpu->iem.s.uCpl != 0)
8527	{
8528	uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
8529	for (uint32_t i = 0; i < cXmmRegs; i++)
8530	pDst->aXMM[i] = pSrc->aXMM[i];
8531	/** @todo Testcase: What happens to the reserved XMM registers? Untouched,
8532	* right? */
8533	}
8534
8535	/*
8536	* Commit the memory.
8537	*/
8538	rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE);
8539	if (rcStrict != VINF_SUCCESS)
8540	return rcStrict;
8541
8542	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8543	}
8544
8545
8546	/**
8547	* Implements 'FXRSTOR'.
8548	*
8549	* @param iEffSeg The effective segment register for @a GCPtrEff.
8550	* @param GCPtrEff The address of the image.
8551	* @param enmEffOpSize The operand size (only REX.W really matters).
8552	*/
8553	IEM_CIMPL_DEF_3(iemCImpl_fxrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
8554	{
8555	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX);
8556
8557	/*
8558	* Raise exceptions.
8559	*/
8560	if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_TS \| X86_CR0_EM))
8561	return iemRaiseDeviceNotAvailable(pVCpu);
8562
8563	/*
8564	* Access the memory.
8565	*/
8566	void *pvMem512;
8567	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_R,
8568	15 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_GP_OR_AC);
8569	if (rcStrict != VINF_SUCCESS)
8570	return rcStrict;
8571	PCX86FXSTATE pSrc = (PCX86FXSTATE)pvMem512;
8572	PX86FXSTATE pDst = &pVCpu->cpum.GstCtx.XState.x87;
8573
8574	/*
8575	* Check the state for stuff which will #GP(0).
8576	*/
8577	uint32_t const fMXCSR = pSrc->MXCSR;
8578	uint32_t const fMXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
8579	if (fMXCSR & ~fMXCSR_MASK)
8580	{
8581	Log(("fxrstor: MXCSR=%#x (MXCSR_MASK=%#x) -> #GP(0)\n", fMXCSR, fMXCSR_MASK));
8582	return iemRaiseGeneralProtectionFault0(pVCpu);
8583	}
8584
8585	/*
8586	* Load the registers.
8587	*/
8588	/** @todo CPU/VM detection possible! If CR4.OSFXSR=0 MXCSR it's
8589	* implementation specific whether MXCSR and XMM0-XMM7 are restored. */
8590
8591	/* common for all formats */
8592	pDst->FCW = pSrc->FCW;
8593	pDst->FSW = pSrc->FSW;
8594	pDst->FTW = pSrc->FTW & UINT16_C(0xff);
8595	pDst->FOP = pSrc->FOP;
8596	pDst->MXCSR = fMXCSR;
8597	/* (MXCSR_MASK is read-only) */
8598	for (uint32_t i = 0; i < RT_ELEMENTS(pSrc->aRegs); i++)
8599	{
8600	pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
8601	pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
8602	pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
8603	pDst->aRegs[i].au32[3] = 0;
8604	}
8605
8606	/* FPU IP, CS, DP and DS. */
8607	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
8608	{
8609	pDst->FPUIP = pSrc->FPUIP;
8610	pDst->CS = pSrc->CS;
8611	pDst->Rsrvd1 = pSrc->Rsrvd1;
8612	pDst->FPUDP = pSrc->FPUDP;
8613	pDst->DS = pSrc->DS;
8614	pDst->Rsrvd2 = pSrc->Rsrvd2;
8615	}
8616	else
8617	{
8618	pDst->FPUIP = pSrc->FPUIP;
8619	pDst->CS = pSrc->CS;
8620	pDst->Rsrvd1 = 0;
8621	pDst->FPUDP = pSrc->FPUDP;
8622	pDst->DS = pSrc->DS;
8623	pDst->Rsrvd2 = 0;
8624	}
8625
8626	/* XMM registers. */
8627	if ( !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_FFXSR)
8628	\|\| pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
8629	\|\| pVCpu->iem.s.uCpl != 0)
8630	{
8631	uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
8632	for (uint32_t i = 0; i < cXmmRegs; i++)
8633	pDst->aXMM[i] = pSrc->aXMM[i];
8634	}
8635
8636	if (pDst->FSW & X86_FSW_ES)
8637	Log11(("fxrstor: %04x:%08RX64: loading state with pending FPU exception (FSW=%#x)\n",
8638	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pSrc->FSW));
8639
8640	/*
8641	* Commit the memory.
8642	*/
8643	rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_R);
8644	if (rcStrict != VINF_SUCCESS)
8645	return rcStrict;
8646
8647	iemHlpUsedFpu(pVCpu);
8648	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8649	}
8650
8651
8652	/**
8653	* Implements 'XSAVE'.
8654	*
8655	* @param iEffSeg The effective segment.
8656	* @param GCPtrEff The address of the image.
8657	* @param enmEffOpSize The operand size (only REX.W really matters).
8658	*/
8659	IEM_CIMPL_DEF_3(iemCImpl_xsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
8660	{
8661	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
8662
8663	/*
8664	* Raise exceptions.
8665	*/
8666	if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
8667	return iemRaiseUndefinedOpcode(pVCpu);
8668	/* When in VMX non-root mode and XSAVE/XRSTOR is not enabled, it results in #UD. */
8669	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
8670	&& !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_XSAVES_XRSTORS))
8671	{
8672	Log(("xrstor: Not enabled for nested-guest execution -> #UD\n"));
8673	return iemRaiseUndefinedOpcode(pVCpu);
8674	}
8675	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)
8676	return iemRaiseDeviceNotAvailable(pVCpu);
8677
8678	/*
8679	* Calc the requested mask.
8680	*/
8681	uint64_t const fReqComponents = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx) & pVCpu->cpum.GstCtx.aXcr[0];
8682	AssertLogRelReturn(!(fReqComponents & ~(XSAVE_C_X87 \| XSAVE_C_SSE \| XSAVE_C_YMM)), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
8683	uint64_t const fXInUse = pVCpu->cpum.GstCtx.aXcr[0];
8684
8685	/** @todo figure out the exact protocol for the memory access. Currently we
8686	* just need this crap to work halfways to make it possible to test
8687	* AVX instructions. */
8688	/** @todo figure out the XINUSE and XMODIFIED */
8689
8690	/*
8691	* Access the x87 memory state.
8692	*/
8693	/* The x87+SSE state. */
8694	void *pvMem512;
8695	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE,
8696	63 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_GP_OR_AC);
8697	if (rcStrict != VINF_SUCCESS)
8698	return rcStrict;
8699	PX86FXSTATE pDst = (PX86FXSTATE)pvMem512;
8700	PCX86FXSTATE pSrc = &pVCpu->cpum.GstCtx.XState.x87;
8701
8702	/* The header. */
8703	PX86XSAVEHDR pHdr;
8704	rcStrict = iemMemMap(pVCpu, (void *)&pHdr, sizeof(&pHdr), iEffSeg, GCPtrEff + 512, IEM_ACCESS_DATA_RW, 0 / checked above */);
8705	if (rcStrict != VINF_SUCCESS)
8706	return rcStrict;
8707
8708	/*
8709	* Store the X87 state.
8710	*/
8711	if (fReqComponents & XSAVE_C_X87)
8712	{
8713	/* common for all formats */
8714	pDst->FCW = pSrc->FCW;
8715	pDst->FSW = pSrc->FSW;
8716	pDst->FTW = pSrc->FTW & UINT16_C(0xff);
8717	pDst->FOP = pSrc->FOP;
8718	pDst->FPUIP = pSrc->FPUIP;
8719	pDst->CS = pSrc->CS;
8720	pDst->FPUDP = pSrc->FPUDP;
8721	pDst->DS = pSrc->DS;
8722	if (enmEffOpSize == IEMMODE_64BIT)
8723	{
8724	/* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
8725	pDst->Rsrvd1 = pSrc->Rsrvd1;
8726	pDst->Rsrvd2 = pSrc->Rsrvd2;
8727	}
8728	else
8729	{
8730	pDst->Rsrvd1 = 0;
8731	pDst->Rsrvd2 = 0;
8732	}
8733	for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
8734	{
8735	/** @todo Testcase: What actually happens to the 6 reserved bytes? I'm clearing
8736	* them for now... */
8737	pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
8738	pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
8739	pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
8740	pDst->aRegs[i].au32[3] = 0;
8741	}
8742
8743	}
8744
8745	if (fReqComponents & (XSAVE_C_SSE \| XSAVE_C_YMM))
8746	{
8747	pDst->MXCSR = pSrc->MXCSR;
8748	pDst->MXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
8749	}
8750
8751	if (fReqComponents & XSAVE_C_SSE)
8752	{
8753	/* XMM registers. */
8754	uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
8755	for (uint32_t i = 0; i < cXmmRegs; i++)
8756	pDst->aXMM[i] = pSrc->aXMM[i];
8757	/** @todo Testcase: What happens to the reserved XMM registers? Untouched,
8758	* right? */
8759	}
8760
8761	/* Commit the x87 state bits. (probably wrong) */
8762	rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE);
8763	if (rcStrict != VINF_SUCCESS)
8764	return rcStrict;
8765
8766	/*
8767	* Store AVX state.
8768	*/
8769	if (fReqComponents & XSAVE_C_YMM)
8770	{
8771	/** @todo testcase: xsave64 vs xsave32 wrt XSAVE_C_YMM. */
8772	AssertLogRelReturn(pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT] != UINT16_MAX, VERR_IEM_IPE_9);
8773	PCX86XSAVEYMMHI pCompSrc = CPUMCTX_XSAVE_C_PTR(IEM_GET_CTX(pVCpu), XSAVE_C_YMM_BIT, PCX86XSAVEYMMHI);
8774	PX86XSAVEYMMHI pCompDst;
8775	rcStrict = iemMemMap(pVCpu, (void *)&pCompDst, sizeof(pCompDst), iEffSeg, GCPtrEff + pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT],
8776	IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE, 0 /* checked above */);
8777	if (rcStrict != VINF_SUCCESS)
8778	return rcStrict;
8779
8780	uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
8781	for (uint32_t i = 0; i < cXmmRegs; i++)
8782	pCompDst->aYmmHi[i] = pCompSrc->aYmmHi[i];
8783
8784	rcStrict = iemMemCommitAndUnmap(pVCpu, pCompDst, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE);
8785	if (rcStrict != VINF_SUCCESS)
8786	return rcStrict;
8787	}
8788
8789	/*
8790	* Update the header.
8791	*/
8792	pHdr->bmXState = (pHdr->bmXState & ~fReqComponents)
8793	\| (fReqComponents & fXInUse);
8794
8795	rcStrict = iemMemCommitAndUnmap(pVCpu, pHdr, IEM_ACCESS_DATA_RW);
8796	if (rcStrict != VINF_SUCCESS)
8797	return rcStrict;
8798
8799	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8800	}
8801
8802
8803	/**
8804	* Implements 'XRSTOR'.
8805	*
8806	* @param iEffSeg The effective segment.
8807	* @param GCPtrEff The address of the image.
8808	* @param enmEffOpSize The operand size (only REX.W really matters).
8809	*/
8810	IEM_CIMPL_DEF_3(iemCImpl_xrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
8811	{
8812	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
8813
8814	/*
8815	* Raise exceptions.
8816	*/
8817	if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
8818	return iemRaiseUndefinedOpcode(pVCpu);
8819	/* When in VMX non-root mode and XSAVE/XRSTOR is not enabled, it results in #UD. */
8820	if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
8821	&& !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_XSAVES_XRSTORS))
8822	{
8823	Log(("xrstor: Not enabled for nested-guest execution -> #UD\n"));
8824	return iemRaiseUndefinedOpcode(pVCpu);
8825	}
8826	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)
8827	return iemRaiseDeviceNotAvailable(pVCpu);
8828	if (GCPtrEff & 63)
8829	{
8830	/** @todo CPU/VM detection possible! \#AC might not be signal for
8831	* all/any misalignment sizes, intel says its an implementation detail. */
8832	if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
8833	&& pVCpu->cpum.GstCtx.eflags.Bits.u1AC
8834	&& pVCpu->iem.s.uCpl == 3)
8835	return iemRaiseAlignmentCheckException(pVCpu);
8836	return iemRaiseGeneralProtectionFault0(pVCpu);
8837	}
8838
8839	/** @todo figure out the exact protocol for the memory access. Currently we
8840	* just need this crap to work halfways to make it possible to test
8841	* AVX instructions. */
8842	/** @todo figure out the XINUSE and XMODIFIED */
8843
8844	/*
8845	* Access the x87 memory state.
8846	*/
8847	/* The x87+SSE state. */
8848	void *pvMem512;
8849	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_R,
8850	63 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_GP_OR_AC);
8851	if (rcStrict != VINF_SUCCESS)
8852	return rcStrict;
8853	PCX86FXSTATE pSrc = (PCX86FXSTATE)pvMem512;
8854	PX86FXSTATE pDst = &pVCpu->cpum.GstCtx.XState.x87;
8855
8856	/*
8857	* Calc the requested mask
8858	*/
8859	PX86XSAVEHDR pHdrDst = &pVCpu->cpum.GstCtx.XState.Hdr;
8860	PCX86XSAVEHDR pHdrSrc;
8861	rcStrict = iemMemMap(pVCpu, (void **)&pHdrSrc, sizeof(&pHdrSrc), iEffSeg, GCPtrEff + 512,
8862	IEM_ACCESS_DATA_R, 0 /* checked above */);
8863	if (rcStrict != VINF_SUCCESS)
8864	return rcStrict;
8865
8866	uint64_t const fReqComponents = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx) & pVCpu->cpum.GstCtx.aXcr[0];
8867	AssertLogRelReturn(!(fReqComponents & ~(XSAVE_C_X87 \| XSAVE_C_SSE \| XSAVE_C_YMM)), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
8868	//uint64_t const fXInUse = pVCpu->cpum.GstCtx.aXcr[0];
8869	uint64_t const fRstorMask = pHdrSrc->bmXState;
8870	uint64_t const fCompMask = pHdrSrc->bmXComp;
8871
8872	AssertLogRelReturn(!(fCompMask & XSAVE_C_X), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
8873
8874	uint32_t const cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
8875
8876	/* We won't need this any longer. */
8877	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pHdrSrc, IEM_ACCESS_DATA_R);
8878	if (rcStrict != VINF_SUCCESS)
8879	return rcStrict;
8880
8881	/*
8882	* Store the X87 state.
8883	*/
8884	if (fReqComponents & XSAVE_C_X87)
8885	{
8886	if (fRstorMask & XSAVE_C_X87)
8887	{
8888	pDst->FCW = pSrc->FCW;
8889	pDst->FSW = pSrc->FSW;
8890	pDst->FTW = pSrc->FTW & UINT16_C(0xff);
8891	pDst->FOP = pSrc->FOP;
8892	pDst->FPUIP = pSrc->FPUIP;
8893	pDst->CS = pSrc->CS;
8894	pDst->FPUDP = pSrc->FPUDP;
8895	pDst->DS = pSrc->DS;
8896	if (enmEffOpSize == IEMMODE_64BIT)
8897	{
8898	/* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
8899	pDst->Rsrvd1 = pSrc->Rsrvd1;
8900	pDst->Rsrvd2 = pSrc->Rsrvd2;
8901	}
8902	else
8903	{
8904	pDst->Rsrvd1 = 0;
8905	pDst->Rsrvd2 = 0;
8906	}
8907	for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
8908	{
8909	pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
8910	pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
8911	pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
8912	pDst->aRegs[i].au32[3] = 0;
8913	}
8914	if (pDst->FSW & X86_FSW_ES)
8915	Log11(("xrstor: %04x:%08RX64: loading state with pending FPU exception (FSW=%#x)\n",
8916	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pSrc->FSW));
8917	}
8918	else
8919	{
8920	pDst->FCW = 0x37f;
8921	pDst->FSW = 0;
8922	pDst->FTW = 0x00; /* 0 - empty. */
8923	pDst->FPUDP = 0;
8924	pDst->DS = 0; //??
8925	pDst->Rsrvd2= 0;
8926	pDst->FPUIP = 0;
8927	pDst->CS = 0; //??
8928	pDst->Rsrvd1= 0;
8929	pDst->FOP = 0;
8930	for (uint32_t i = 0; i < RT_ELEMENTS(pSrc->aRegs); i++)
8931	{
8932	pDst->aRegs[i].au32[0] = 0;
8933	pDst->aRegs[i].au32[1] = 0;
8934	pDst->aRegs[i].au32[2] = 0;
8935	pDst->aRegs[i].au32[3] = 0;
8936	}
8937	}
8938	pHdrDst->bmXState \|= XSAVE_C_X87; /* playing safe for now */
8939	}
8940
8941	/* MXCSR */
8942	if (fReqComponents & (XSAVE_C_SSE \| XSAVE_C_YMM))
8943	{
8944	if (fRstorMask & (XSAVE_C_SSE \| XSAVE_C_YMM))
8945	pDst->MXCSR = pSrc->MXCSR;
8946	else
8947	pDst->MXCSR = 0x1f80;
8948	}
8949
8950	/* XMM registers. */
8951	if (fReqComponents & XSAVE_C_SSE)
8952	{
8953	if (fRstorMask & XSAVE_C_SSE)
8954	{
8955	for (uint32_t i = 0; i < cXmmRegs; i++)
8956	pDst->aXMM[i] = pSrc->aXMM[i];
8957	/** @todo Testcase: What happens to the reserved XMM registers? Untouched,
8958	* right? */
8959	}
8960	else
8961	{
8962	for (uint32_t i = 0; i < cXmmRegs; i++)
8963	{
8964	pDst->aXMM[i].au64[0] = 0;
8965	pDst->aXMM[i].au64[1] = 0;
8966	}
8967	}
8968	pHdrDst->bmXState \|= XSAVE_C_SSE; /* playing safe for now */
8969	}
8970
8971	/* Unmap the x87 state bits (so we've don't run out of mapping). */
8972	rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_R);
8973	if (rcStrict != VINF_SUCCESS)
8974	return rcStrict;
8975
8976	/*
8977	* Restore AVX state.
8978	*/
8979	if (fReqComponents & XSAVE_C_YMM)
8980	{
8981	AssertLogRelReturn(pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT] != UINT16_MAX, VERR_IEM_IPE_9);
8982	PX86XSAVEYMMHI pCompDst = CPUMCTX_XSAVE_C_PTR(IEM_GET_CTX(pVCpu), XSAVE_C_YMM_BIT, PX86XSAVEYMMHI);
8983
8984	if (fRstorMask & XSAVE_C_YMM)
8985	{
8986	/** @todo testcase: xsave64 vs xsave32 wrt XSAVE_C_YMM. */
8987	PCX86XSAVEYMMHI pCompSrc;
8988	rcStrict = iemMemMap(pVCpu, (void *)&pCompSrc, sizeof(pCompDst),
8989	iEffSeg, GCPtrEff + pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT],
8990	IEM_ACCESS_DATA_R, 0 /* checked above */);
8991	if (rcStrict != VINF_SUCCESS)
8992	return rcStrict;
8993
8994	for (uint32_t i = 0; i < cXmmRegs; i++)
8995	{
8996	pCompDst->aYmmHi[i].au64[0] = pCompSrc->aYmmHi[i].au64[0];
8997	pCompDst->aYmmHi[i].au64[1] = pCompSrc->aYmmHi[i].au64[1];
8998	}
8999
9000	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pCompSrc, IEM_ACCESS_DATA_R);
9001	if (rcStrict != VINF_SUCCESS)
9002	return rcStrict;
9003	}
9004	else
9005	{
9006	for (uint32_t i = 0; i < cXmmRegs; i++)
9007	{
9008	pCompDst->aYmmHi[i].au64[0] = 0;
9009	pCompDst->aYmmHi[i].au64[1] = 0;
9010	}
9011	}
9012	pHdrDst->bmXState \|= XSAVE_C_YMM; /* playing safe for now */
9013	}
9014
9015	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9016	}
9017
9018
9019
9020
9021	/**
9022	* Implements 'STMXCSR'.
9023	*
9024	* @param iEffSeg The effective segment register for @a GCPtrEff.
9025	* @param GCPtrEff The address of the image.
9026	*/
9027	IEM_CIMPL_DEF_2(iemCImpl_stmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
9028	{
9029	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX);
9030
9031	/*
9032	* Raise exceptions.
9033	*/
9034	if ( !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
9035	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR))
9036	{
9037	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
9038	{
9039	/*
9040	* Do the job.
9041	*/
9042	VBOXSTRICTRC rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEff, pVCpu->cpum.GstCtx.XState.x87.MXCSR);
9043	if (rcStrict == VINF_SUCCESS)
9044	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9045	return rcStrict;
9046	}
9047	return iemRaiseDeviceNotAvailable(pVCpu);
9048	}
9049	return iemRaiseUndefinedOpcode(pVCpu);
9050	}
9051
9052
9053	/**
9054	* Implements 'VSTMXCSR'.
9055	*
9056	* @param iEffSeg The effective segment register for @a GCPtrEff.
9057	* @param GCPtrEff The address of the image.
9058	*/
9059	IEM_CIMPL_DEF_2(iemCImpl_vstmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
9060	{
9061	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_XCRx);
9062
9063	/*
9064	* Raise exceptions.
9065	*/
9066	if ( ( !IEM_IS_GUEST_CPU_AMD(pVCpu)
9067	? (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_SSE \| XSAVE_C_YMM)) == (XSAVE_C_SSE \| XSAVE_C_YMM)
9068	: !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)) /* AMD Jaguar CPU (f0x16,m0,s1) behaviour */
9069	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
9070	{
9071	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
9072	{
9073	/*
9074	* Do the job.
9075	*/
9076	VBOXSTRICTRC rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEff, pVCpu->cpum.GstCtx.XState.x87.MXCSR);
9077	if (rcStrict == VINF_SUCCESS)
9078	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9079	return rcStrict;
9080	}
9081	return iemRaiseDeviceNotAvailable(pVCpu);
9082	}
9083	return iemRaiseUndefinedOpcode(pVCpu);
9084	}
9085
9086
9087	/**
9088	* Implements 'LDMXCSR'.
9089	*
9090	* @param iEffSeg The effective segment register for @a GCPtrEff.
9091	* @param GCPtrEff The address of the image.
9092	*/
9093	IEM_CIMPL_DEF_2(iemCImpl_ldmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
9094	{
9095	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX);
9096
9097	/*
9098	* Raise exceptions.
9099	*/
9100	/** @todo testcase - order of LDMXCSR faults. Does \#PF, \#GP and \#SS
9101	* happen after or before \#UD and \#EM? */
9102	if ( !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
9103	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR))
9104	{
9105	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
9106	{
9107	/*
9108	* Do the job.
9109	*/
9110	uint32_t fNewMxCsr;
9111	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, &fNewMxCsr, iEffSeg, GCPtrEff);
9112	if (rcStrict == VINF_SUCCESS)
9113	{
9114	uint32_t const fMxCsrMask = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
9115	if (!(fNewMxCsr & ~fMxCsrMask))
9116	{
9117	pVCpu->cpum.GstCtx.XState.x87.MXCSR = fNewMxCsr;
9118	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9119	}
9120	Log(("ldmxcsr: New MXCSR=%#RX32 & ~MASK=%#RX32 = %#RX32 -> #GP(0)\n",
9121	fNewMxCsr, fMxCsrMask, fNewMxCsr & ~fMxCsrMask));
9122	return iemRaiseGeneralProtectionFault0(pVCpu);
9123	}
9124	return rcStrict;
9125	}
9126	return iemRaiseDeviceNotAvailable(pVCpu);
9127	}
9128	return iemRaiseUndefinedOpcode(pVCpu);
9129	}
9130
9131
9132	/**
9133	* Commmon routine for fnstenv and fnsave.
9134	*
9135	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9136	* @param enmEffOpSize The effective operand size.
9137	* @param uPtr Where to store the state.
9138	*/
9139	static void iemCImplCommonFpuStoreEnv(PVMCPUCC pVCpu, IEMMODE enmEffOpSize, RTPTRUNION uPtr)
9140	{
9141	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9142	PCX86FXSTATE pSrcX87 = &pVCpu->cpum.GstCtx.XState.x87;
9143	if (enmEffOpSize == IEMMODE_16BIT)
9144	{
9145	uPtr.pu16[0] = pSrcX87->FCW;
9146	uPtr.pu16[1] = pSrcX87->FSW;
9147	uPtr.pu16[2] = iemFpuCalcFullFtw(pSrcX87);
9148	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
9149	{
9150	/** @todo Testcase: How does this work when the FPUIP/CS was saved in
9151	* protected mode or long mode and we save it in real mode? And vice
9152	* versa? And with 32-bit operand size? I think CPU is storing the
9153	* effective address ((CS << 4) + IP) in the offset register and not
9154	* doing any address calculations here. */
9155	uPtr.pu16[3] = (uint16_t)pSrcX87->FPUIP;
9156	uPtr.pu16[4] = ((pSrcX87->FPUIP >> 4) & UINT16_C(0xf000)) \| pSrcX87->FOP;
9157	uPtr.pu16[5] = (uint16_t)pSrcX87->FPUDP;
9158	uPtr.pu16[6] = (pSrcX87->FPUDP >> 4) & UINT16_C(0xf000);
9159	}
9160	else
9161	{
9162	uPtr.pu16[3] = pSrcX87->FPUIP;
9163	uPtr.pu16[4] = pSrcX87->CS;
9164	uPtr.pu16[5] = pSrcX87->FPUDP;
9165	uPtr.pu16[6] = pSrcX87->DS;
9166	}
9167	}
9168	else
9169	{
9170	/** @todo Testcase: what is stored in the "gray" areas? (figure 8-9 and 8-10) */
9171	uPtr.pu16[0*2] = pSrcX87->FCW;
9172	uPtr.pu16[02+1] = 0xffff; / (0xffff observed on intel skylake.) */
9173	uPtr.pu16[1*2] = pSrcX87->FSW;
9174	uPtr.pu16[1*2+1] = 0xffff;
9175	uPtr.pu16[2*2] = iemFpuCalcFullFtw(pSrcX87);
9176	uPtr.pu16[2*2+1] = 0xffff;
9177	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
9178	{
9179	uPtr.pu16[3*2] = (uint16_t)pSrcX87->FPUIP;
9180	uPtr.pu32[4] = ((pSrcX87->FPUIP & UINT32_C(0xffff0000)) >> 4) \| pSrcX87->FOP;
9181	uPtr.pu16[5*2] = (uint16_t)pSrcX87->FPUDP;
9182	uPtr.pu32[6] = (pSrcX87->FPUDP & UINT32_C(0xffff0000)) >> 4;
9183	}
9184	else
9185	{
9186	uPtr.pu32[3] = pSrcX87->FPUIP;
9187	uPtr.pu16[4*2] = pSrcX87->CS;
9188	uPtr.pu16[4*2+1] = pSrcX87->FOP;
9189	uPtr.pu32[5] = pSrcX87->FPUDP;
9190	uPtr.pu16[6*2] = pSrcX87->DS;
9191	uPtr.pu16[6*2+1] = 0xffff;
9192	}
9193	}
9194	}
9195
9196
9197	/**
9198	* Commmon routine for fldenv and frstor
9199	*
9200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9201	* @param enmEffOpSize The effective operand size.
9202	* @param uPtr Where to store the state.
9203	*/
9204	static void iemCImplCommonFpuRestoreEnv(PVMCPUCC pVCpu, IEMMODE enmEffOpSize, RTCPTRUNION uPtr)
9205	{
9206	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9207	PX86FXSTATE pDstX87 = &pVCpu->cpum.GstCtx.XState.x87;
9208	if (enmEffOpSize == IEMMODE_16BIT)
9209	{
9210	pDstX87->FCW = uPtr.pu16[0];
9211	pDstX87->FSW = uPtr.pu16[1];
9212	pDstX87->FTW = uPtr.pu16[2];
9213	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
9214	{
9215	pDstX87->FPUIP = uPtr.pu16[3] \| ((uint32_t)(uPtr.pu16[4] & UINT16_C(0xf000)) << 4);
9216	pDstX87->FPUDP = uPtr.pu16[5] \| ((uint32_t)(uPtr.pu16[6] & UINT16_C(0xf000)) << 4);
9217	pDstX87->FOP = uPtr.pu16[4] & UINT16_C(0x07ff);
9218	pDstX87->CS = 0;
9219	pDstX87->Rsrvd1= 0;
9220	pDstX87->DS = 0;
9221	pDstX87->Rsrvd2= 0;
9222	}
9223	else
9224	{
9225	pDstX87->FPUIP = uPtr.pu16[3];
9226	pDstX87->CS = uPtr.pu16[4];
9227	pDstX87->Rsrvd1= 0;
9228	pDstX87->FPUDP = uPtr.pu16[5];
9229	pDstX87->DS = uPtr.pu16[6];
9230	pDstX87->Rsrvd2= 0;
9231	/** @todo Testcase: Is FOP cleared when doing 16-bit protected mode fldenv? */
9232	}
9233	}
9234	else
9235	{
9236	pDstX87->FCW = uPtr.pu16[0*2];
9237	pDstX87->FSW = uPtr.pu16[1*2];
9238	pDstX87->FTW = uPtr.pu16[2*2];
9239	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
9240	{
9241	pDstX87->FPUIP = uPtr.pu16[3*2] \| ((uPtr.pu32[4] & UINT32_C(0x0ffff000)) << 4);
9242	pDstX87->FOP = uPtr.pu32[4] & UINT16_C(0x07ff);
9243	pDstX87->FPUDP = uPtr.pu16[5*2] \| ((uPtr.pu32[6] & UINT32_C(0x0ffff000)) << 4);
9244	pDstX87->CS = 0;
9245	pDstX87->Rsrvd1= 0;
9246	pDstX87->DS = 0;
9247	pDstX87->Rsrvd2= 0;
9248	}
9249	else
9250	{
9251	pDstX87->FPUIP = uPtr.pu32[3];
9252	pDstX87->CS = uPtr.pu16[4*2];
9253	pDstX87->Rsrvd1= 0;
9254	pDstX87->FOP = uPtr.pu16[4*2+1];
9255	pDstX87->FPUDP = uPtr.pu32[5];
9256	pDstX87->DS = uPtr.pu16[6*2];
9257	pDstX87->Rsrvd2= 0;
9258	}
9259	}
9260
9261	/* Make adjustments. */
9262	pDstX87->FTW = iemFpuCompressFtw(pDstX87->FTW);
9263	#ifdef LOG_ENABLED
9264	uint16_t const fOldFsw = pDstX87->FSW;
9265	#endif
9266	pDstX87->FCW &= ~X86_FCW_ZERO_MASK;
9267	iemFpuRecalcExceptionStatus(pDstX87);
9268	#ifdef LOG_ENABLED
9269	if ((pDstX87->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES))
9270	Log11(("iemCImplCommonFpuRestoreEnv: %04x:%08RX64: %s FPU exception (FCW=%#x FSW=%#x -> %#x)\n",
9271	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fOldFsw & X86_FSW_ES ? "Supressed" : "Raised",
9272	pDstX87->FCW, fOldFsw, pDstX87->FSW));
9273	#endif
9274
9275	/** @todo Testcase: Check if ES and/or B are automatically cleared if no
9276	* exceptions are pending after loading the saved state? */
9277	}
9278
9279
9280	/**
9281	* Implements 'FNSTENV'.
9282	*
9283	* @param enmEffOpSize The operand size (only REX.W really matters).
9284	* @param iEffSeg The effective segment register for @a GCPtrEffDst.
9285	* @param GCPtrEffDst The address of the image.
9286	*/
9287	IEM_CIMPL_DEF_3(iemCImpl_fnstenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
9288	{
9289	RTPTRUNION uPtr;
9290	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 14 : 28,
9291	iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE,
9292	enmEffOpSize == IEMMODE_16BIT ? 1 : 3 /** @todo ? */);
9293	if (rcStrict != VINF_SUCCESS)
9294	return rcStrict;
9295
9296	iemCImplCommonFpuStoreEnv(pVCpu, enmEffOpSize, uPtr);
9297
9298	rcStrict = iemMemCommitAndUnmap(pVCpu, uPtr.pv, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE);
9299	if (rcStrict != VINF_SUCCESS)
9300	return rcStrict;
9301
9302	/* Mask all math exceptions. Any possibly pending exceptions will be cleared. */
9303	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9304	pFpuCtx->FCW \|= X86_FCW_XCPT_MASK;
9305	#ifdef LOG_ENABLED
9306	uint16_t fOldFsw = pFpuCtx->FSW;
9307	#endif
9308	iemFpuRecalcExceptionStatus(pFpuCtx);
9309	#ifdef LOG_ENABLED
9310	if ((pFpuCtx->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES))
9311	Log11(("fnstenv: %04x:%08RX64: %s FPU exception (FCW=%#x, FSW %#x -> %#x)\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9312	fOldFsw & X86_FSW_ES ? "Supressed" : "Raised", pFpuCtx->FCW, fOldFsw, pFpuCtx->FSW));
9313	#endif
9314
9315	iemHlpUsedFpu(pVCpu);
9316
9317	/* Note: C0, C1, C2 and C3 are documented as undefined, we leave them untouched! */
9318	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9319	}
9320
9321
9322	/**
9323	* Implements 'FNSAVE'.
9324	*
9325	* @param enmEffOpSize The operand size.
9326	* @param iEffSeg The effective segment register for @a GCPtrEffDst.
9327	* @param GCPtrEffDst The address of the image.
9328	*/
9329	IEM_CIMPL_DEF_3(iemCImpl_fnsave, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
9330	{
9331	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9332
9333	RTPTRUNION uPtr;
9334	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 94 : 108,
9335	iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE, 3 /** @todo ? */);
9336	if (rcStrict != VINF_SUCCESS)
9337	return rcStrict;
9338
9339	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9340	iemCImplCommonFpuStoreEnv(pVCpu, enmEffOpSize, uPtr);
9341	PRTFLOAT80U paRegs = (PRTFLOAT80U)(uPtr.pu8 + (enmEffOpSize == IEMMODE_16BIT ? 14 : 28));
9342	for (uint32_t i = 0; i < RT_ELEMENTS(pFpuCtx->aRegs); i++)
9343	{
9344	paRegs[i].au32[0] = pFpuCtx->aRegs[i].au32[0];
9345	paRegs[i].au32[1] = pFpuCtx->aRegs[i].au32[1];
9346	paRegs[i].au16[4] = pFpuCtx->aRegs[i].au16[4];
9347	}
9348
9349	rcStrict = iemMemCommitAndUnmap(pVCpu, uPtr.pv, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE);
9350	if (rcStrict != VINF_SUCCESS)
9351	return rcStrict;
9352
9353	/* Rotate the stack to account for changed TOS. */
9354	iemFpuRotateStackSetTop(pFpuCtx, 0);
9355
9356	/*
9357	* Re-initialize the FPU context.
9358	*/
9359	pFpuCtx->FCW = 0x37f;
9360	pFpuCtx->FSW = 0;
9361	pFpuCtx->FTW = 0x00; /* 0 - empty */
9362	pFpuCtx->FPUDP = 0;
9363	pFpuCtx->DS = 0;
9364	pFpuCtx->Rsrvd2= 0;
9365	pFpuCtx->FPUIP = 0;
9366	pFpuCtx->CS = 0;
9367	pFpuCtx->Rsrvd1= 0;
9368	pFpuCtx->FOP = 0;
9369
9370	iemHlpUsedFpu(pVCpu);
9371	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9372	}
9373
9374
9375
9376	/**
9377	* Implements 'FLDENV'.
9378	*
9379	* @param enmEffOpSize The operand size (only REX.W really matters).
9380	* @param iEffSeg The effective segment register for @a GCPtrEffSrc.
9381	* @param GCPtrEffSrc The address of the image.
9382	*/
9383	IEM_CIMPL_DEF_3(iemCImpl_fldenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
9384	{
9385	RTCPTRUNION uPtr;
9386	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 14 : 28,
9387	iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R,
9388	enmEffOpSize == IEMMODE_16BIT ? 1 : 3 /** @todo ?*/);
9389	if (rcStrict != VINF_SUCCESS)
9390	return rcStrict;
9391
9392	iemCImplCommonFpuRestoreEnv(pVCpu, enmEffOpSize, uPtr);
9393
9394	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uPtr.pv, IEM_ACCESS_DATA_R);
9395	if (rcStrict != VINF_SUCCESS)
9396	return rcStrict;
9397
9398	iemHlpUsedFpu(pVCpu);
9399	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9400	}
9401
9402
9403	/**
9404	* Implements 'FRSTOR'.
9405	*
9406	* @param enmEffOpSize The operand size.
9407	* @param iEffSeg The effective segment register for @a GCPtrEffSrc.
9408	* @param GCPtrEffSrc The address of the image.
9409	*/
9410	IEM_CIMPL_DEF_3(iemCImpl_frstor, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
9411	{
9412	RTCPTRUNION uPtr;
9413	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 94 : 108,
9414	iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R, 3 /** @todo ?*/ );
9415	if (rcStrict != VINF_SUCCESS)
9416	return rcStrict;
9417
9418	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9419	iemCImplCommonFpuRestoreEnv(pVCpu, enmEffOpSize, uPtr);
9420	PCRTFLOAT80U paRegs = (PCRTFLOAT80U)(uPtr.pu8 + (enmEffOpSize == IEMMODE_16BIT ? 14 : 28));
9421	for (uint32_t i = 0; i < RT_ELEMENTS(pFpuCtx->aRegs); i++)
9422	{
9423	pFpuCtx->aRegs[i].au32[0] = paRegs[i].au32[0];
9424	pFpuCtx->aRegs[i].au32[1] = paRegs[i].au32[1];
9425	pFpuCtx->aRegs[i].au32[2] = paRegs[i].au16[4];
9426	pFpuCtx->aRegs[i].au32[3] = 0;
9427	}
9428
9429	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uPtr.pv, IEM_ACCESS_DATA_R);
9430	if (rcStrict != VINF_SUCCESS)
9431	return rcStrict;
9432
9433	iemHlpUsedFpu(pVCpu);
9434	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9435	}
9436
9437
9438	/**
9439	* Implements 'FLDCW'.
9440	*
9441	* @param u16Fcw The new FCW.
9442	*/
9443	IEM_CIMPL_DEF_1(iemCImpl_fldcw, uint16_t, u16Fcw)
9444	{
9445	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9446
9447	/** @todo Testcase: Check what happens when trying to load X86_FCW_PC_RSVD. */
9448	/** @todo Testcase: Try see what happens when trying to set undefined bits
9449	* (other than 6 and 7). Currently ignoring them. */
9450	/** @todo Testcase: Test that it raises and loweres the FPU exception bits
9451	* according to FSW. (This is what is currently implemented.) */
9452	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9453	pFpuCtx->FCW = u16Fcw & ~X86_FCW_ZERO_MASK;
9454	#ifdef LOG_ENABLED
9455	uint16_t fOldFsw = pFpuCtx->FSW;
9456	#endif
9457	iemFpuRecalcExceptionStatus(pFpuCtx);
9458	#ifdef LOG_ENABLED
9459	if ((pFpuCtx->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES))
9460	Log11(("fldcw: %04x:%08RX64: %s FPU exception (FCW=%#x, FSW %#x -> %#x)\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9461	fOldFsw & X86_FSW_ES ? "Supressed" : "Raised", pFpuCtx->FCW, fOldFsw, pFpuCtx->FSW));
9462	#endif
9463
9464	/* Note: C0, C1, C2 and C3 are documented as undefined, we leave them untouched! */
9465	iemHlpUsedFpu(pVCpu);
9466	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9467	}
9468
9469
9470
9471	/**
9472	* Implements the underflow case of fxch.
9473	*
9474	* @param iStReg The other stack register.
9475	*/
9476	IEM_CIMPL_DEF_1(iemCImpl_fxch_underflow, uint8_t, iStReg)
9477	{
9478	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9479
9480	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9481	unsigned const iReg1 = X86_FSW_TOP_GET(pFpuCtx->FSW);
9482	unsigned const iReg2 = (iReg1 + iStReg) & X86_FSW_TOP_SMASK;
9483	Assert(!(RT_BIT(iReg1) & pFpuCtx->FTW) \|\| !(RT_BIT(iReg2) & pFpuCtx->FTW));
9484
9485	/** @todo Testcase: fxch underflow. Making assumptions that underflowed
9486	* registers are read as QNaN and then exchanged. This could be
9487	* wrong... */
9488	if (pFpuCtx->FCW & X86_FCW_IM)
9489	{
9490	if (RT_BIT(iReg1) & pFpuCtx->FTW)
9491	{
9492	if (RT_BIT(iReg2) & pFpuCtx->FTW)
9493	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
9494	else
9495	pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[iStReg].r80;
9496	iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
9497	}
9498	else
9499	{
9500	pFpuCtx->aRegs[iStReg].r80 = pFpuCtx->aRegs[0].r80;
9501	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
9502	}
9503	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
9504	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
9505	}
9506	else
9507	{
9508	/* raise underflow exception, don't change anything. */
9509	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_XCPT_MASK);
9510	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
9511	Log11(("fxch: %04x:%08RX64: Underflow exception (FSW=%#x)\n",
9512	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
9513	}
9514
9515	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
9516	iemHlpUsedFpu(pVCpu);
9517	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9518	}
9519
9520
9521	/**
9522	* Implements 'FCOMI', 'FCOMIP', 'FUCOMI', and 'FUCOMIP'.
9523	*
9524	* @param iStReg The other stack register.
9525	* @param pfnAImpl The assembly comparison implementation.
9526	* @param fPop Whether we should pop the stack when done or not.
9527	*/
9528	IEM_CIMPL_DEF_3(iemCImpl_fcomi_fucomi, uint8_t, iStReg, PFNIEMAIMPLFPUR80EFL, pfnAImpl, bool, fPop)
9529	{
9530	Assert(iStReg < 8);
9531	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9532
9533	/*
9534	* Raise exceptions.
9535	*/
9536	if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS))
9537	return iemRaiseDeviceNotAvailable(pVCpu);
9538
9539	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9540	uint16_t u16Fsw = pFpuCtx->FSW;
9541	if (u16Fsw & X86_FSW_ES)
9542	return iemRaiseMathFault(pVCpu);
9543
9544	/*
9545	* Check if any of the register accesses causes #SF + #IA.
9546	*/
9547	unsigned const iReg1 = X86_FSW_TOP_GET(u16Fsw);
9548	unsigned const iReg2 = (iReg1 + iStReg) & X86_FSW_TOP_SMASK;
9549	if ((pFpuCtx->FTW & (RT_BIT(iReg1) \| RT_BIT(iReg2))) == (RT_BIT(iReg1) \| RT_BIT(iReg2)))
9550	{
9551	uint32_t u32Eflags = pfnAImpl(pFpuCtx, &u16Fsw, &pFpuCtx->aRegs[0].r80, &pFpuCtx->aRegs[iStReg].r80);
9552
9553	pFpuCtx->FSW &= ~X86_FSW_C1;
9554	pFpuCtx->FSW \|= u16Fsw & ~X86_FSW_TOP_MASK;
9555	if ( !(u16Fsw & X86_FSW_IE)
9556	\|\| (pFpuCtx->FCW & X86_FCW_IM) )
9557	{
9558	pVCpu->cpum.GstCtx.eflags.u &= ~(X86_EFL_OF \| X86_EFL_SF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_PF \| X86_EFL_CF);
9559	pVCpu->cpum.GstCtx.eflags.u \|= u32Eflags & (X86_EFL_ZF \| X86_EFL_PF \| X86_EFL_CF);
9560	}
9561	}
9562	else if (pFpuCtx->FCW & X86_FCW_IM)
9563	{
9564	/* Masked underflow. */
9565	pFpuCtx->FSW &= ~X86_FSW_C1;
9566	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
9567	pVCpu->cpum.GstCtx.eflags.u &= ~(X86_EFL_OF \| X86_EFL_SF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_PF \| X86_EFL_CF);
9568	pVCpu->cpum.GstCtx.eflags.u \|= X86_EFL_ZF \| X86_EFL_PF \| X86_EFL_CF;
9569	}
9570	else
9571	{
9572	/* Raise underflow - don't touch EFLAGS or TOP. */
9573	pFpuCtx->FSW &= ~X86_FSW_C1;
9574	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
9575	Log11(("fxch: %04x:%08RX64: Raising IE+SF exception (FSW=%#x)\n",
9576	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
9577	fPop = false;
9578	}
9579
9580	/*
9581	* Pop if necessary.
9582	*/
9583	if (fPop)
9584	{
9585	pFpuCtx->FTW &= ~RT_BIT(iReg1);
9586	iemFpuStackIncTop(pVCpu);
9587	}
9588
9589	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
9590	iemHlpUsedFpu(pVCpu);
9591	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9592	}
9593
9594	/** @} */
9595

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllCImpl.cpp@ 97441

Download in other formats: